@tandryukha/aidemo 0.8.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +5 -2
- package/README.md +16 -4
- package/docs/AUTHORING.md +734 -48
- package/package.json +3 -2
- package/src/anchors.ts +128 -0
- package/src/attention.ts +365 -0
- package/src/caption-render.ts +17 -8
- package/src/captions.ts +244 -36
- package/src/capture.ts +113 -0
- package/src/cards.ts +15 -6
- package/src/cli.ts +711 -26
- package/src/compose.ts +1318 -101
- package/src/cursor-overlay.ts +7 -113
- package/src/cursor.ts +19 -4
- package/src/distribute.ts +37 -4
- package/src/expr.ts +130 -0
- package/src/ffmpeg.ts +224 -0
- package/src/frame.ts +205 -0
- package/src/frames.ts +208 -0
- package/src/golden.ts +27 -0
- package/src/guide.ts +102 -0
- package/src/i18n.ts +7 -4
- package/src/import-trace.ts +640 -0
- package/src/inspect.ts +316 -0
- package/src/lint.ts +673 -0
- package/src/login.ts +143 -0
- package/src/mcp/jobs.ts +5 -1
- package/src/mcp/server.ts +468 -27
- package/src/player.ts +1286 -41
- package/src/profile.ts +128 -0
- package/src/project.ts +12 -2
- package/src/qa.ts +295 -0
- package/src/recorder.ts +316 -21
- package/src/safe-regex.ts +36 -0
- package/src/setup.ts +248 -0
- package/src/starter.ts +244 -0
- package/src/stills.ts +15 -4
- package/src/types.ts +796 -22
- package/src/variants.ts +1 -1
- package/src/voice-local.ts +7 -1
- package/src/voice.ts +60 -1
- package/src/walkthrough.ts +250 -0
- package/src/zoom.ts +85 -39
package/src/compose.ts
CHANGED
|
@@ -1,22 +1,64 @@
|
|
|
1
1
|
import { promises as fs } from "node:fs";
|
|
2
|
-
import { resolve, isAbsolute } from "node:path";
|
|
2
|
+
import { resolve, isAbsolute, relative } from "node:path";
|
|
3
3
|
import { Project } from "./project.js";
|
|
4
|
-
import type {
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
import type {
|
|
5
|
+
Storyboard,
|
|
6
|
+
Timeline,
|
|
7
|
+
IdleSpan,
|
|
8
|
+
Card,
|
|
9
|
+
Output,
|
|
10
|
+
Loudness,
|
|
11
|
+
ComposeReport,
|
|
12
|
+
ComposeSceneReport,
|
|
13
|
+
ComposeWarning,
|
|
14
|
+
} from "./types.js";
|
|
15
|
+
import {
|
|
16
|
+
TimelineSchema,
|
|
17
|
+
VoiceManifestSchema,
|
|
18
|
+
LoudnessSchema,
|
|
19
|
+
HoldSchema,
|
|
20
|
+
AutoIdleSchema,
|
|
21
|
+
} from "./types.js";
|
|
22
|
+
import { sceneWordsFor, type Cue } from "./captions.js";
|
|
23
|
+
import { wordStartMs } from "./anchors.js";
|
|
7
24
|
import { renderCaptionPngs } from "./caption-render.js";
|
|
8
25
|
import { renderCardPng } from "./cards.js";
|
|
9
26
|
import { renderCursorPng } from "./cursor.js";
|
|
10
|
-
import { buildZoomFilter, type ZoomEvent } from "./zoom.js";
|
|
27
|
+
import { buildZoomFilter, clampScaleForWidth, type ZoomEvent } from "./zoom.js";
|
|
11
28
|
import {
|
|
12
29
|
buildCursorFilter,
|
|
13
30
|
type CursorPoint,
|
|
14
31
|
type HideWindow,
|
|
15
32
|
} from "./cursor-overlay.js";
|
|
16
|
-
import {
|
|
33
|
+
import {
|
|
34
|
+
renderAttentionPngs,
|
|
35
|
+
renderKeyChipPngs,
|
|
36
|
+
renderClickRingPngs,
|
|
37
|
+
buildRedactFilter,
|
|
38
|
+
REDACT_BATCH,
|
|
39
|
+
KEY_CHIP_MS,
|
|
40
|
+
DEFAULT_ACCENT,
|
|
41
|
+
type OverlayItem,
|
|
42
|
+
type PlacedAttention,
|
|
43
|
+
type PlacedKey,
|
|
44
|
+
type PlacedClick,
|
|
45
|
+
type PlacedRedact,
|
|
46
|
+
} from "./attention.js";
|
|
47
|
+
import { renderFramePng } from "./frame.js";
|
|
48
|
+
import { OUTPUT_PRESETS, type CaptionsConfig } from "./types.js";
|
|
49
|
+
import {
|
|
50
|
+
runFfmpeg,
|
|
51
|
+
probeDurationMs,
|
|
52
|
+
probeFlatTailMs,
|
|
53
|
+
probeFlatLeadMs,
|
|
54
|
+
probeVideoDims,
|
|
55
|
+
probeFreezeSpans,
|
|
56
|
+
type FreezeSpan,
|
|
57
|
+
} from "./ffmpeg.js";
|
|
17
58
|
import {
|
|
18
59
|
ensureDir,
|
|
19
60
|
readJson,
|
|
61
|
+
writeJson,
|
|
20
62
|
exists,
|
|
21
63
|
log,
|
|
22
64
|
ok,
|
|
@@ -43,14 +85,33 @@ export async function compose(
|
|
|
43
85
|
project: Project,
|
|
44
86
|
storyboard: Storyboard,
|
|
45
87
|
opts: SceneProgress = {}
|
|
46
|
-
): Promise<
|
|
88
|
+
): Promise<ComposeReport> {
|
|
47
89
|
step("Composing final video (ffmpeg)");
|
|
48
90
|
|
|
49
91
|
const timeline = TimelineSchema.parse(await readJson(project.timelinePath));
|
|
50
92
|
const voice = VoiceManifestSchema.parse(
|
|
51
93
|
await readJson(project.voiceManifestPath)
|
|
52
94
|
);
|
|
53
|
-
|
|
95
|
+
// Structured twin of the log lines below — see ComposeReportSchema.
|
|
96
|
+
const warnings: ComposeWarning[] = [];
|
|
97
|
+
const sceneReports: ComposeSceneReport[] = [];
|
|
98
|
+
const warn = (code: string, message: string, scene?: string): void => {
|
|
99
|
+
warnings.push({ code, message, ...(scene ? { scene } : {}) });
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* Frames to dump next to a warning (issue #59): an agent cannot look at the
|
|
103
|
+
* MP4, so "the reward is on screen for 0.3 s" is only actionable with the
|
|
104
|
+
* frame at that moment attached. Content-time ms; the intro shift is added
|
|
105
|
+
* when they are extracted, after the final mux.
|
|
106
|
+
*/
|
|
107
|
+
const warningFrames: Array<{ name: string; atMs: number }> = [];
|
|
108
|
+
const hold = HoldSchema.parse(storyboard.hold ?? {});
|
|
109
|
+
if (await warnStaleCaptions(project)) {
|
|
110
|
+
warn(
|
|
111
|
+
"stale-captions",
|
|
112
|
+
`narration.mp3 is newer than the caption files — re-run: aidemo captions ${project.dir}`
|
|
113
|
+
);
|
|
114
|
+
}
|
|
54
115
|
const tmp = project.composeTmpDir;
|
|
55
116
|
await fs.rm(tmp, { recursive: true, force: true });
|
|
56
117
|
await ensureDir(tmp);
|
|
@@ -63,6 +124,41 @@ export async function compose(
|
|
|
63
124
|
const { width: outW, height: outH } = await probeVideoDims(rawVideo);
|
|
64
125
|
const pxScale = outW / storyboard.video.width;
|
|
65
126
|
|
|
127
|
+
// --- autoIdle / multi-source plumbing -------------------------------------
|
|
128
|
+
// A resumed take splices scenes from more than one raw file; every source
|
|
129
|
+
// must share the main take's pixel size or all the overlay math (captions,
|
|
130
|
+
// zoom, cursor — all scaled by pxScale) silently misaligns on those scenes.
|
|
131
|
+
const dimsChecked = new Set<string>([rawVideo]);
|
|
132
|
+
const assertSourceDims = async (file: string): Promise<void> => {
|
|
133
|
+
if (dimsChecked.has(file)) return;
|
|
134
|
+
dimsChecked.add(file);
|
|
135
|
+
const d = await probeVideoDims(file);
|
|
136
|
+
if (d.width !== outW || d.height !== outH) {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`resumed take mixes recordings of different sizes: ${relative(project.dir, file)} is ` +
|
|
139
|
+
`${d.width}x${d.height}, the current take is ${outW}x${outH} — re-record the whole demo ` +
|
|
140
|
+
`(a resume must use the same capture mode and viewport as the take it continues)`
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
const autoIdleCfg = AutoIdleSchema.parse(
|
|
145
|
+
typeof storyboard.autoIdle === "object" ? storyboard.autoIdle : {}
|
|
146
|
+
);
|
|
147
|
+
const autoIdleOn =
|
|
148
|
+
storyboard.autoIdle === true ||
|
|
149
|
+
(typeof storyboard.autoIdle === "object" && storyboard.autoIdle.enabled !== false);
|
|
150
|
+
const autoIdleFor = (scene: { autoIdle?: boolean } | undefined): boolean =>
|
|
151
|
+
scene?.autoIdle ?? autoIdleOn;
|
|
152
|
+
const freezeCache = new Map<string, Promise<FreezeSpan[]>>();
|
|
153
|
+
const freezeSpansFor = (file: string): Promise<FreezeSpan[]> => {
|
|
154
|
+
let p = freezeCache.get(file);
|
|
155
|
+
if (!p) {
|
|
156
|
+
p = probeFreezeSpans(file, autoIdleCfg.minMs ?? 1500, autoIdleCfg.noise ?? 0.003);
|
|
157
|
+
freezeCache.set(file, p);
|
|
158
|
+
}
|
|
159
|
+
return p;
|
|
160
|
+
};
|
|
161
|
+
|
|
66
162
|
// Map scene id -> narration duration; compose target = narration + gap.
|
|
67
163
|
const voiceById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
|
|
68
164
|
const sceneById = new Map(storyboard.scenes.map((s) => [s.id, s]));
|
|
@@ -80,8 +176,24 @@ export async function compose(
|
|
|
80
176
|
const cursorPts: CursorPoint[] = [];
|
|
81
177
|
const hideWindows: HideWindow[] = [];
|
|
82
178
|
let cursorSampleCount = 0;
|
|
179
|
+
// Attention layer accumulators (all opt-in; empty for a plain storyboard).
|
|
180
|
+
const accent = storyboard.attention?.color ?? storyboard.brand?.accent ?? DEFAULT_ACCENT;
|
|
181
|
+
const attentionPlaced: PlacedAttention[] = [];
|
|
182
|
+
const keysPlaced: PlacedKey[] = [];
|
|
183
|
+
const clicksPlaced: PlacedClick[] = [];
|
|
184
|
+
const redactPlaced: PlacedRedact[] = [];
|
|
185
|
+
const clickRings = !!(storyboard.attention?.clicks && cursorCfg);
|
|
186
|
+
/** Per-scene caption placement windows (content ms, before the intro shift). */
|
|
187
|
+
const captionWindows: Array<{ a: number; b: number; position: "top" | "bottom" }> = [];
|
|
188
|
+
// Dot cursors are centered on the point; the arrow PNG's tip is its top-left.
|
|
189
|
+
const dotOffset =
|
|
190
|
+
cursorCfg?.style === "dot" ? Math.round(16 * pxScale * (cursorCfg.scale ?? 1)) : 0;
|
|
83
191
|
|
|
84
192
|
const sceneVideos: string[] = [];
|
|
193
|
+
/** timeline scenes that actually produced a segment, parallel to sceneVideos. */
|
|
194
|
+
const composedTl: Timeline["scenes"] = [];
|
|
195
|
+
/** content-time span of each composed scene, for `frame.url: "auto"`. */
|
|
196
|
+
const sceneSpans: Array<{ a: number; b: number }> = [];
|
|
85
197
|
const sceneTotal = timeline.scenes.length;
|
|
86
198
|
for (let i = 0; i < timeline.scenes.length; i++) {
|
|
87
199
|
const tl = timeline.scenes[i];
|
|
@@ -90,22 +202,161 @@ export async function compose(
|
|
|
90
202
|
const targetMs = narrMs + voice.gapMs;
|
|
91
203
|
if (targetMs <= 0) {
|
|
92
204
|
log(`scene ${tl.id}: no narration, skipping`);
|
|
205
|
+
warn("scene-no-narration", `scene ${tl.id} has no narration audio and was skipped`, tl.id);
|
|
93
206
|
opts.onSceneComplete?.(tl.id, i, sceneTotal);
|
|
94
207
|
continue;
|
|
95
208
|
}
|
|
96
209
|
|
|
97
|
-
|
|
210
|
+
let blankTrimMs = 0;
|
|
211
|
+
// A resumed take's reused scenes live in an earlier raw file with its own lead-in.
|
|
212
|
+
const leadInMs = tl.leadInMs ?? timeline.leadInMs;
|
|
213
|
+
const sceneRaw = tl.source ? resolve(project.dir, tl.source) : rawVideo;
|
|
214
|
+
await assertSourceDims(sceneRaw);
|
|
215
|
+
// `autoIdle`: motionless spans of the raw take count as idle too, so dead
|
|
216
|
+
// air the storyboard never annotated gets the same cap as a `waitFor`.
|
|
217
|
+
const autoSpans = autoIdleFor(sceneById.get(tl.id))
|
|
218
|
+
? sceneAutoIdle(await freezeSpansFor(sceneRaw), tl, leadInMs)
|
|
219
|
+
: [];
|
|
220
|
+
if (autoSpans.length) {
|
|
221
|
+
const dead = autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0);
|
|
222
|
+
log(`scene ${tl.id}: autoIdle found ${autoSpans.length} motionless span(s), ${dead}ms`);
|
|
223
|
+
}
|
|
224
|
+
const keeps = keepIntervals(
|
|
225
|
+
autoSpans.length ? { ...tl, idleSpans: mergeIdle([...tl.idleSpans, ...autoSpans]) } : tl,
|
|
226
|
+
leadInMs
|
|
227
|
+
);
|
|
98
228
|
const rawSegPath = resolve(tmp, `scene-${i}-raw.mp4`);
|
|
99
|
-
await extractAndConcat(
|
|
229
|
+
await extractAndConcat(sceneRaw, keeps, rawSegPath, tmp, i);
|
|
100
230
|
|
|
101
|
-
|
|
231
|
+
let srcMs = await probeDurationMs(rawSegPath);
|
|
232
|
+
// #47: the take opens on the browser's white pre-paint (the `goto` load),
|
|
233
|
+
// so the first caption plays over a blank page and `poster` grabs white.
|
|
234
|
+
// Trim it down to a sliver — opt-in, so an unchanged storyboard renders
|
|
235
|
+
// byte-for-byte as before.
|
|
236
|
+
let leadTrimMs = 0;
|
|
237
|
+
if (i === 0 && storyboard.output?.trimLeadingBlank && srcMs > 1000) {
|
|
238
|
+
const cap = storyboard.output.blankOpenCapMs ?? 100;
|
|
239
|
+
const flatLead = await probeFlatLeadMs(rawSegPath);
|
|
240
|
+
if (flatLead > cap + 80) {
|
|
241
|
+
leadTrimMs = Math.min(flatLead - cap, srcMs - 500);
|
|
242
|
+
srcMs -= leadTrimMs;
|
|
243
|
+
log(
|
|
244
|
+
`scene ${tl.id}: trimmed ${Math.round(leadTrimMs)}ms of blank opening ` +
|
|
245
|
+
`(kept ${cap}ms of it)`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
// A scene that ends right before a `goto` can trail off into the browser's
|
|
250
|
+
// white pre-paint. Freeze-holding THAT frame paints seconds of solid white
|
|
251
|
+
// (issue #36), so drop a flat tail before deciding how long to hold —
|
|
252
|
+
// but only when we would actually hold, and never the whole segment.
|
|
253
|
+
if (srcMs > 0 && targetMs > srcMs + 400) {
|
|
254
|
+
const flatMs = await probeFlatTailMs(rawSegPath, srcMs);
|
|
255
|
+
const trimTo = srcMs - flatMs;
|
|
256
|
+
if (flatMs > 200 && trimTo > 500) {
|
|
257
|
+
blankTrimMs = flatMs;
|
|
258
|
+
srcMs = trimTo;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Raw video ms → offset inside this scene's kept, head-trimmed footage.
|
|
263
|
+
* Every event remap goes through it, so the leading-blank trim shifts the
|
|
264
|
+
* whole scene's timeline consistently.
|
|
265
|
+
*/
|
|
266
|
+
const localIn = (rawT: number): number =>
|
|
267
|
+
Math.max(0, offsetInKeeps(keeps, rawT) - leadTrimMs);
|
|
102
268
|
// Retime the segment toward the narration length, but only within
|
|
103
269
|
// [MIN_FACTOR, MAX_STRETCH]. If narration still needs more time, hold
|
|
104
270
|
// (freeze) the last frame for the remainder — natural for a static page,
|
|
105
271
|
// and far better than 3x slow-motion.
|
|
106
272
|
const ratio = targetMs / Math.max(srcMs, 1);
|
|
107
|
-
const
|
|
108
|
-
|
|
273
|
+
const uniformFactor = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, ratio));
|
|
274
|
+
|
|
275
|
+
// Narration anchors → piecewise retime. Each anchored action is a raw
|
|
276
|
+
// offset (inside the kept spans) that must land on the ms its word is
|
|
277
|
+
// spoken; between anchors the factor is whatever gets there (clamped to
|
|
278
|
+
// the same [MIN_FACTOR, MAX_STRETCH], so a beat that can't be reached is
|
|
279
|
+
// reported, never faked). No anchors → one piece, the uniform factor —
|
|
280
|
+
// arithmetically identical to the pre-anchor path.
|
|
281
|
+
const sceneDef = sceneById.get(tl.id);
|
|
282
|
+
const anchorDefs = sceneDef?.anchors ?? {};
|
|
283
|
+
const anchorPts: Array<{ name: string; raw: number; target: number }> = [];
|
|
284
|
+
if (Object.keys(anchorDefs).length && (tl.anchorEvents ?? []).length) {
|
|
285
|
+
const words = await sceneWordsFor(project, tl.id, sceneDef?.narration ?? "", narrMs);
|
|
286
|
+
for (const ev of tl.anchorEvents) {
|
|
287
|
+
const wi = anchorDefs[ev.name];
|
|
288
|
+
if (wi == null) continue;
|
|
289
|
+
const target = wordStartMs(words, sceneDef?.narration ?? "", wi);
|
|
290
|
+
if (target == null) continue;
|
|
291
|
+
const raw = localIn(ev.tMs + leadInMs);
|
|
292
|
+
if (raw > srcMs - 40) {
|
|
293
|
+
warn(
|
|
294
|
+
"anchor-unreachable",
|
|
295
|
+
`scene ${tl.id}: anchor "${ev.name}" fires at the very end of the take (${Math.round(raw)}ms of ${Math.round(srcMs)}ms) — add a pause or a later action after it so the beat has room`,
|
|
296
|
+
tl.id
|
|
297
|
+
);
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
anchorPts.push({ name: ev.name, raw, target });
|
|
301
|
+
}
|
|
302
|
+
anchorPts.sort((a, b) => a.raw - b.raw);
|
|
303
|
+
}
|
|
304
|
+
const pieces: RetimePiece[] = [];
|
|
305
|
+
const anchorReport: NonNullable<ComposeSceneReport["anchors"]> = [];
|
|
306
|
+
if (anchorPts.length) {
|
|
307
|
+
let out = 0;
|
|
308
|
+
let prevRaw = 0;
|
|
309
|
+
const bounds = [
|
|
310
|
+
...anchorPts.map((a) => ({ raw: a.raw, target: a.target, name: a.name as string | null })),
|
|
311
|
+
{ raw: srcMs, target: targetMs, name: null as string | null },
|
|
312
|
+
];
|
|
313
|
+
for (const bnd of bounds) {
|
|
314
|
+
const len = bnd.raw - prevRaw;
|
|
315
|
+
if (len >= 40) {
|
|
316
|
+
const want = bnd.target - out;
|
|
317
|
+
const f = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, want / len));
|
|
318
|
+
pieces.push({ a: prevRaw, b: bnd.raw, f, out });
|
|
319
|
+
out += len * f;
|
|
320
|
+
prevRaw = bnd.raw;
|
|
321
|
+
}
|
|
322
|
+
if (bnd.name) {
|
|
323
|
+
const offMs = Math.round(out - bnd.target);
|
|
324
|
+
anchorReport.push({
|
|
325
|
+
name: bnd.name,
|
|
326
|
+
targetMs: Math.round(bnd.target),
|
|
327
|
+
landedMs: Math.round(out),
|
|
328
|
+
offMs,
|
|
329
|
+
});
|
|
330
|
+
if (Math.abs(offMs) > 150) {
|
|
331
|
+
warningFrames.push({
|
|
332
|
+
name: `${tl.id}-${bnd.name}`,
|
|
333
|
+
atMs: outCursorMs + Math.max(0, Math.min(out, targetMs)),
|
|
334
|
+
});
|
|
335
|
+
warn(
|
|
336
|
+
"anchor-unreachable",
|
|
337
|
+
`scene ${tl.id}: anchor "${bnd.name}" lands ${Math.abs(offMs)}ms ${offMs > 0 ? "after" : "before"} its word even at ${offMs > 0 ? "2x speed" : "x1.6 slow-motion"} — ${
|
|
338
|
+
offMs > 0
|
|
339
|
+
? "trim what happens before the action (mark waits idle, drop a scroll) or move the {{@" + bnd.name + "}} marker later"
|
|
340
|
+
: "give the action more lead-in (a pause, a hover) or move the {{@" + bnd.name + "}} marker earlier"
|
|
341
|
+
}`,
|
|
342
|
+
tl.id
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
if (!pieces.length) pieces.push({ a: 0, b: srcMs, f: uniformFactor, out: 0 });
|
|
349
|
+
const anchored = anchorPts.length > 0;
|
|
350
|
+
let stretchedMs = anchored
|
|
351
|
+
? pieces[pieces.length - 1].out + (pieces[pieces.length - 1].b - pieces[pieces.length - 1].a) * pieces[pieces.length - 1].f
|
|
352
|
+
: uniformFactor * srcMs;
|
|
353
|
+
const factor = anchored ? stretchedMs / Math.max(srcMs, 1) : uniformFactor;
|
|
354
|
+
/** Kept-span offset (raw ms inside the concatenated keeps) → stretched scene ms. */
|
|
355
|
+
const stretchLocal = (local: number): number => {
|
|
356
|
+
let pc = pieces[0];
|
|
357
|
+
for (const q of pieces) if (local >= q.a) pc = q;
|
|
358
|
+
return pc.out + (local - pc.a) * pc.f;
|
|
359
|
+
};
|
|
109
360
|
// A scene whose active video exceeds 2x its narration would OVERRUN its
|
|
110
361
|
// narration slot — the narration track is fixed, so every later scene's
|
|
111
362
|
// video drifts behind its audio and the finale slides past the end of the
|
|
@@ -118,20 +369,77 @@ export async function compose(
|
|
|
118
369
|
stretchedMs = targetMs;
|
|
119
370
|
}
|
|
120
371
|
const holdMs = Math.max(0, targetMs - stretchedMs);
|
|
121
|
-
|
|
372
|
+
const holding = holdMs > 40;
|
|
373
|
+
// `hold.backoffMs` takes the held frame from slightly before the segment
|
|
374
|
+
// end (a mid-flight hover, a half-painted transition); the hold grows by
|
|
375
|
+
// the same amount so the scene still fills its narration slot exactly.
|
|
376
|
+
const backoffMs = holding
|
|
377
|
+
? Math.min(hold.backoffMs, Math.max(0, stretchedMs - 500))
|
|
378
|
+
: 0;
|
|
379
|
+
// Drift only where it reads as a camera move (issue #57): a short hold
|
|
380
|
+
// bobs, and inside a composited window chrome the page appears to slide
|
|
381
|
+
// within a frame that does not move with it.
|
|
382
|
+
const framedChrome =
|
|
383
|
+
!!storyboard.frame && (storyboard.frame.chrome ?? "none") !== "none";
|
|
384
|
+
const driftSuppressed =
|
|
385
|
+
holding && hold.mode === "drift"
|
|
386
|
+
? holdMs < hold.minDriftMs
|
|
387
|
+
? `hold is ${Math.round(holdMs)}ms (< hold.minDriftMs ${hold.minDriftMs}ms)`
|
|
388
|
+
: framedChrome && !hold.driftInFrame
|
|
389
|
+
? `frame.chrome "${storyboard.frame!.chrome}" is static — the drift would read as the page sliding inside it (set hold.driftInFrame to override)`
|
|
390
|
+
: null
|
|
391
|
+
: null;
|
|
392
|
+
if (driftSuppressed) log(`scene ${tl.id}: drift hold → freeze (${driftSuppressed})`);
|
|
393
|
+
const driftHold = holding && hold.mode === "drift" && !driftSuppressed;
|
|
394
|
+
let vf = "";
|
|
395
|
+
if (leadTrimMs > 0) {
|
|
396
|
+
vf += `trim=start=${(leadTrimMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
|
|
397
|
+
}
|
|
398
|
+
if (blankTrimMs > 0) {
|
|
399
|
+
vf += `trim=duration=${(srcMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
|
|
400
|
+
}
|
|
401
|
+
// Tail (shared by the uniform and the piecewise path): overrun trim,
|
|
402
|
+
// backoff trim, freeze hold.
|
|
403
|
+
let tail = "";
|
|
122
404
|
if (overrunTrimMs > 0) {
|
|
123
|
-
|
|
405
|
+
tail += `,trim=duration=${(targetMs / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
|
|
406
|
+
}
|
|
407
|
+
if (backoffMs > 0) {
|
|
408
|
+
tail += `,trim=duration=${((stretchedMs - backoffMs) / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
|
|
124
409
|
}
|
|
125
|
-
if (
|
|
126
|
-
|
|
410
|
+
if (holding && !driftHold) {
|
|
411
|
+
tail += `,tpad=stop_mode=clone:stop_duration=${((holdMs + backoffMs) / 1000).toFixed(3)}`;
|
|
412
|
+
}
|
|
413
|
+
let filterArgs: string[];
|
|
414
|
+
if (anchored) {
|
|
415
|
+
// split → per-piece trim + setpts → concat. Same source, same format,
|
|
416
|
+
// so concat is a plain join; everything stays in the core filter set.
|
|
417
|
+
const n = pieces.length;
|
|
418
|
+
const splitLabels = pieces.map((_, k) => `[s${k}]`).join("");
|
|
419
|
+
const pieceFilters = pieces
|
|
420
|
+
.map(
|
|
421
|
+
(pc, k) =>
|
|
422
|
+
`[s${k}]trim=start=${(pc.a / 1000).toFixed(3)}:end=${(pc.b / 1000).toFixed(3)},` +
|
|
423
|
+
`setpts=(PTS-STARTPTS)*${pc.f.toFixed(6)}[p${k}]`
|
|
424
|
+
)
|
|
425
|
+
.join(";");
|
|
426
|
+
const concatIn = pieces.map((_, k) => `[p${k}]`).join("");
|
|
427
|
+
const fc =
|
|
428
|
+
`[0:v]${vf}split=${n}${splitLabels};${pieceFilters};` +
|
|
429
|
+
`${concatIn}concat=n=${n}:v=1:a=0,setpts=PTS-STARTPTS${tail}[vout]`;
|
|
430
|
+
filterArgs = ["-filter_complex", fc, "-map", "[vout]"];
|
|
431
|
+
} else {
|
|
432
|
+
filterArgs = ["-vf", `${vf}setpts=${factor.toFixed(6)}*PTS${tail}`];
|
|
127
433
|
}
|
|
128
434
|
|
|
129
435
|
// Map this scene's focus events into final-video time for the zoom pass:
|
|
130
436
|
// raw time → offset inside the kept spans → stretched → + scene offset.
|
|
437
|
+
let sceneFocus = 0;
|
|
131
438
|
if (zoomCfg && sceneById.get(tl.id)?.zoom !== false) {
|
|
132
439
|
for (const ev of tl.focusEvents ?? []) {
|
|
133
|
-
|
|
134
|
-
const
|
|
440
|
+
sceneFocus++;
|
|
441
|
+
const rawT = ev.tMs + leadInMs;
|
|
442
|
+
const local = stretchLocal(localIn(rawT));
|
|
135
443
|
zoomEvents.push({
|
|
136
444
|
tMs: outCursorMs + Math.min(local, stretchedMs),
|
|
137
445
|
x: ev.x * pxScale,
|
|
@@ -148,12 +456,12 @@ export async function compose(
|
|
|
148
456
|
if (cursorCfg) {
|
|
149
457
|
const sceneStart = outCursorMs;
|
|
150
458
|
for (const s of tl.cursorSamples ?? []) {
|
|
151
|
-
const rawT = s.tMs +
|
|
152
|
-
const local =
|
|
459
|
+
const rawT = s.tMs + leadInMs;
|
|
460
|
+
const local = stretchLocal(localIn(rawT));
|
|
153
461
|
cursorPts.push({
|
|
154
462
|
t: (outCursorMs + Math.min(local, stretchedMs)) / 1000,
|
|
155
|
-
x: s.x * pxScale,
|
|
156
|
-
y: s.y * pxScale,
|
|
463
|
+
x: s.x * pxScale - dotOffset,
|
|
464
|
+
y: s.y * pxScale - dotOffset,
|
|
157
465
|
});
|
|
158
466
|
}
|
|
159
467
|
cursorSampleCount += (tl.cursorSamples ?? []).length;
|
|
@@ -163,14 +471,50 @@ export async function compose(
|
|
|
163
471
|
}
|
|
164
472
|
}
|
|
165
473
|
|
|
474
|
+
// Attention beats, key chips, click rings and redact spans: same raw →
|
|
475
|
+
// content-time remap; holds are content-time and clipped to the scene.
|
|
476
|
+
{
|
|
477
|
+
const sceneStart = outCursorMs;
|
|
478
|
+
const sceneEnd = outCursorMs + stretchedMs + (holdMs > 40 ? holdMs : 0);
|
|
479
|
+
const toContent = (rawMs: number): number =>
|
|
480
|
+
sceneStart + Math.min(stretchLocal(localIn(rawMs + leadInMs)), stretchedMs);
|
|
481
|
+
for (const ev of tl.attentionEvents ?? []) {
|
|
482
|
+
const a = toContent(ev.tMs);
|
|
483
|
+
const b = Math.min(sceneEnd, a + ev.holdMs);
|
|
484
|
+
if (b - a < 80) continue;
|
|
485
|
+
attentionPlaced.push({ ...ev, a: a / 1000, b: b / 1000 });
|
|
486
|
+
}
|
|
487
|
+
for (const k of tl.keyEvents ?? []) {
|
|
488
|
+
const a = toContent(k.tMs);
|
|
489
|
+
const b = Math.min(sceneEnd, a + KEY_CHIP_MS);
|
|
490
|
+
if (b - a < 80) continue;
|
|
491
|
+
keysPlaced.push({ keys: k.keys, a: a / 1000, b: b / 1000 });
|
|
492
|
+
}
|
|
493
|
+
if (clickRings) {
|
|
494
|
+
for (const f of tl.focusEvents ?? []) {
|
|
495
|
+
if (f.kind !== "click") continue;
|
|
496
|
+
clicksPlaced.push({ x: f.x, y: f.y, t: toContent(f.tMs) / 1000 });
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
for (const r of tl.redactSpans ?? []) {
|
|
500
|
+
const a = toContent(r.startMs);
|
|
501
|
+
// A span open at the scene end covers the hold too.
|
|
502
|
+
const b = r.endMs >= tl.endMs - 5 ? sceneEnd : toContent(r.endMs);
|
|
503
|
+
if (b - a < 40) continue;
|
|
504
|
+
redactPlaced.push({ ...r, a: a / 1000, b: b / 1000 });
|
|
505
|
+
}
|
|
506
|
+
const pos = sceneById.get(tl.id)?.captions?.position;
|
|
507
|
+
if (pos) captionWindows.push({ a: sceneStart, b: sceneEnd, position: pos });
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const sceneContentStartMs = outCursorMs;
|
|
166
511
|
outCursorMs += stretchedMs + (holdMs > 40 ? holdMs : 0);
|
|
167
512
|
|
|
168
|
-
|
|
513
|
+
let scenePath = resolve(tmp, `scene-${i}.mp4`);
|
|
169
514
|
await runFfmpeg([
|
|
170
515
|
"-i",
|
|
171
516
|
rawSegPath,
|
|
172
|
-
|
|
173
|
-
vf,
|
|
517
|
+
...filterArgs,
|
|
174
518
|
"-an",
|
|
175
519
|
"-r",
|
|
176
520
|
String(FPS),
|
|
@@ -184,13 +528,89 @@ export async function compose(
|
|
|
184
528
|
"yuv420p",
|
|
185
529
|
scenePath,
|
|
186
530
|
]);
|
|
531
|
+
if (driftHold) {
|
|
532
|
+
scenePath = await appendDriftHold(
|
|
533
|
+
scenePath,
|
|
534
|
+
holdMs + backoffMs,
|
|
535
|
+
hold.driftScale,
|
|
536
|
+
outW,
|
|
537
|
+
outH,
|
|
538
|
+
tmp,
|
|
539
|
+
i
|
|
540
|
+
);
|
|
541
|
+
}
|
|
187
542
|
sceneVideos.push(scenePath);
|
|
543
|
+
composedTl.push(tl);
|
|
544
|
+
sceneSpans.push({ a: sceneContentStartMs, b: outCursorMs });
|
|
545
|
+
const holdPct = holding ? holdMs / targetMs : 0;
|
|
188
546
|
log(
|
|
189
547
|
`scene ${tl.id}: ${srcMs}ms -> ${targetMs}ms ` +
|
|
190
|
-
`(x${factor.toFixed(2)}${
|
|
548
|
+
`(x${factor.toFixed(2)}${
|
|
549
|
+
holding
|
|
550
|
+
? ` + ${Math.round(holdMs)}ms ${driftHold ? "drift " : ""}hold` +
|
|
551
|
+
(backoffMs > 0 ? ` (backoff ${backoffMs}ms)` : "")
|
|
552
|
+
: ""
|
|
553
|
+
}${
|
|
191
554
|
overrunTrimMs > 0 ? ` - ${Math.round(overrunTrimMs)}ms tail trim` : ""
|
|
192
|
-
}, ${keeps.length} span(s)
|
|
555
|
+
}${blankTrimMs > 0 ? ` - ${Math.round(blankTrimMs)}ms blank tail` : ""}, ${keeps.length} span(s)${
|
|
556
|
+
anchored
|
|
557
|
+
? `, ${pieces.length} piece(s): ` +
|
|
558
|
+
anchorReport.map((a) => `${a.name} ${a.offMs >= 0 ? "+" : ""}${a.offMs}ms`).join(" ")
|
|
559
|
+
: ""
|
|
560
|
+
})`
|
|
193
561
|
);
|
|
562
|
+
sceneReports.push({
|
|
563
|
+
id: tl.id,
|
|
564
|
+
srcMs: Math.round(srcMs),
|
|
565
|
+
targetMs: Math.round(targetMs),
|
|
566
|
+
factor: Number(factor.toFixed(3)),
|
|
567
|
+
holdMs: Math.round(holding ? holdMs : 0),
|
|
568
|
+
holdPct: Number(holdPct.toFixed(3)),
|
|
569
|
+
tailTrimMs: Math.round(overrunTrimMs),
|
|
570
|
+
blankTrimMs: Math.round(blankTrimMs),
|
|
571
|
+
autoIdleMs: Math.round(autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0)),
|
|
572
|
+
spans: keeps.length,
|
|
573
|
+
focusEvents: sceneFocus,
|
|
574
|
+
...(anchorReport.length ? { anchors: anchorReport } : {}),
|
|
575
|
+
});
|
|
576
|
+
if (holdPct > FREEZE_WARN_PCT) {
|
|
577
|
+
// Name the exact frame being held (issue #55): when the wrong one is
|
|
578
|
+
// frozen — the next page's load, a half-painted card — the only way to
|
|
579
|
+
// see it today is to extract frames by hand.
|
|
580
|
+
const heldRaw = rawAtKeepOffset(
|
|
581
|
+
keeps,
|
|
582
|
+
leadTrimMs + Math.max(0, srcMs - backoffMs / Math.max(factor, 0.01))
|
|
583
|
+
);
|
|
584
|
+
warningFrames.push({
|
|
585
|
+
name: `${tl.id}-held`,
|
|
586
|
+
atMs: sceneContentStartMs + stretchedMs + 200,
|
|
587
|
+
});
|
|
588
|
+
warn(
|
|
589
|
+
"scene-freeze",
|
|
590
|
+
`scene ${tl.id}: ${Math.round(holdPct * 100)}% of its ${(targetMs / 1000).toFixed(1)}s ` +
|
|
591
|
+
`is a held frame (${(srcMs / 1000).toFixed(1)}s of action for ` +
|
|
592
|
+
`${(targetMs / 1000).toFixed(1)}s of narration; the held frame is ` +
|
|
593
|
+
`${relative(project.dir, sceneRaw)} @ ${stampMs(heldRaw)}) — add on-screen action ` +
|
|
594
|
+
`(hover/scroll/focus beats, a pause) or shorten the narration`,
|
|
595
|
+
tl.id
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
if (overrunTrimMs > 0) {
|
|
599
|
+
warn(
|
|
600
|
+
"overrun-trim",
|
|
601
|
+
`scene ${tl.id}: ${Math.round(overrunTrimMs)}ms of recorded action was cut from ` +
|
|
602
|
+
`the tail — the action outran the narration even at 2x; mark waits idle or ` +
|
|
603
|
+
`lengthen the narration`,
|
|
604
|
+
tl.id
|
|
605
|
+
);
|
|
606
|
+
}
|
|
607
|
+
if (blankTrimMs > 0) {
|
|
608
|
+
warn(
|
|
609
|
+
"blank-tail",
|
|
610
|
+
`scene ${tl.id}: dropped a ${Math.round(blankTrimMs)}ms solid-color tail before holding`,
|
|
611
|
+
tl.id
|
|
612
|
+
);
|
|
613
|
+
}
|
|
194
614
|
opts.onSceneComplete?.(tl.id, i, sceneTotal);
|
|
195
615
|
}
|
|
196
616
|
|
|
@@ -201,14 +621,43 @@ export async function compose(
|
|
|
201
621
|
// scene boundaries (re-encodes the join; total duration is preserved).
|
|
202
622
|
let content: string;
|
|
203
623
|
if (storyboard.transition && sceneVideos.length >= 2) {
|
|
204
|
-
const
|
|
624
|
+
const baseDur = storyboard.transition.durationMs;
|
|
625
|
+
// Per-boundary duration; 0 = hard cut. A boundary that crosses a page
|
|
626
|
+
// navigation dissolves the old page over a still-painting new one — a
|
|
627
|
+
// ghost double-image (issue #49) — so cut those, and honour an explicit
|
|
628
|
+
// per-scene override either way.
|
|
629
|
+
const durations: number[] = [];
|
|
630
|
+
const autoCut: string[] = [];
|
|
631
|
+
for (let b = 1; b < sceneVideos.length; b++) {
|
|
632
|
+
const scene = sceneById.get(composedTl[b].id);
|
|
633
|
+
if (scene?.transition === false) {
|
|
634
|
+
durations.push(0);
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
if (scene?.transition) {
|
|
638
|
+
durations.push(scene.transition.durationMs);
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
if (crossesNavigation(composedTl[b - 1], composedTl[b])) {
|
|
642
|
+
durations.push(0);
|
|
643
|
+
autoCut.push(composedTl[b].id);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
durations.push(baseDur);
|
|
647
|
+
}
|
|
648
|
+
const fades = durations.filter((d) => d > 0).length;
|
|
205
649
|
log(
|
|
206
|
-
`scene crossfade: ${
|
|
650
|
+
`scene crossfade: ${baseDur}ms x ${fades} boundary/ies` +
|
|
651
|
+
(durations.length - fades
|
|
652
|
+
? `, ${durations.length - fades} hard cut(s)` +
|
|
653
|
+
(autoCut.length ? ` (navigation into ${autoCut.join(", ")})` : "")
|
|
654
|
+
: "")
|
|
207
655
|
);
|
|
208
656
|
content = await crossfadeScenes(
|
|
209
657
|
sceneVideos,
|
|
210
|
-
|
|
211
|
-
resolve(tmp, "content.mp4")
|
|
658
|
+
durations,
|
|
659
|
+
resolve(tmp, "content.mp4"),
|
|
660
|
+
tmp
|
|
212
661
|
);
|
|
213
662
|
} else {
|
|
214
663
|
content = await concatSegments(
|
|
@@ -219,6 +668,66 @@ export async function compose(
|
|
|
219
668
|
);
|
|
220
669
|
}
|
|
221
670
|
|
|
671
|
+
// Redaction first: blur boxes ride every later pass (cursor, zoom, blur).
|
|
672
|
+
if (redactPlaced.length) {
|
|
673
|
+
let src = content;
|
|
674
|
+
let passes = 0;
|
|
675
|
+
for (let i = 0; i < redactPlaced.length; i += REDACT_BATCH) {
|
|
676
|
+
const batch = redactPlaced.slice(i, i + REDACT_BATCH);
|
|
677
|
+
const filter = buildRedactFilter(batch, outW, outH, pxScale);
|
|
678
|
+
if (!filter) continue;
|
|
679
|
+
const out = resolve(tmp, `content-redact-${passes}.mp4`);
|
|
680
|
+
await runFfmpeg([
|
|
681
|
+
"-i",
|
|
682
|
+
src,
|
|
683
|
+
"-filter_complex",
|
|
684
|
+
filter,
|
|
685
|
+
"-map",
|
|
686
|
+
"[vout]",
|
|
687
|
+
"-an",
|
|
688
|
+
"-r",
|
|
689
|
+
String(FPS),
|
|
690
|
+
"-c:v",
|
|
691
|
+
"libx264",
|
|
692
|
+
"-preset",
|
|
693
|
+
"veryfast",
|
|
694
|
+
"-crf",
|
|
695
|
+
"20",
|
|
696
|
+
"-pix_fmt",
|
|
697
|
+
"yuv420p",
|
|
698
|
+
out,
|
|
699
|
+
]);
|
|
700
|
+
src = out;
|
|
701
|
+
passes++;
|
|
702
|
+
}
|
|
703
|
+
content = src;
|
|
704
|
+
log(`redact: ${redactPlaced.length} span(s) blurred in ${passes} pass(es)`);
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// Attention overlays (highlight / spotlight / callout / click rings) go on
|
|
708
|
+
// BEFORE the cursor and the zoom, so they ride the camera like page content.
|
|
709
|
+
let attentionItems: OverlayItem[] = [];
|
|
710
|
+
if (attentionPlaced.length || clicksPlaced.length) {
|
|
711
|
+
attentionItems = [
|
|
712
|
+
...(await renderAttentionPngs(
|
|
713
|
+
attentionPlaced,
|
|
714
|
+
resolve(tmp, "attention"),
|
|
715
|
+
storyboard.video.width,
|
|
716
|
+
storyboard.video.height,
|
|
717
|
+
pxScale,
|
|
718
|
+
accent
|
|
719
|
+
)),
|
|
720
|
+
...(await renderClickRingPngs(clicksPlaced, resolve(tmp, "attention"), pxScale, accent)),
|
|
721
|
+
];
|
|
722
|
+
const r = await overlayPngBatches(content, attentionItems, tmp, "attention");
|
|
723
|
+
content = r.video;
|
|
724
|
+
log(
|
|
725
|
+
`attention: ${attentionPlaced.length} beat(s)` +
|
|
726
|
+
(clicksPlaced.length ? `, ${clicksPlaced.length} click ring(s)` : "") +
|
|
727
|
+
` in ${r.passes} pass(es)`
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
|
|
222
731
|
// Compose-time cursor overlay (opt-in `cursor` block). Draw the recorded
|
|
223
732
|
// cursor path onto the (cursor-free) content BEFORE the zoom pass, so the
|
|
224
733
|
// cursor zooms and pans with the frame exactly like a baked one would.
|
|
@@ -230,6 +739,10 @@ export async function compose(
|
|
|
230
739
|
"has no recorded cursor path — re-run `record`/`render` with the cursor " +
|
|
231
740
|
"block present (the current recording may still show a baked cursor)."
|
|
232
741
|
);
|
|
742
|
+
warn(
|
|
743
|
+
"cursor-missing",
|
|
744
|
+
"cursor block present but the take has no recorded cursor path — re-record with the block"
|
|
745
|
+
);
|
|
233
746
|
} else if (cursorCfg.hidden) {
|
|
234
747
|
log("cursor: hidden (no overlay drawn)");
|
|
235
748
|
} else {
|
|
@@ -237,7 +750,17 @@ export async function compose(
|
|
|
237
750
|
if (filter) {
|
|
238
751
|
const cScale = cursorCfg.scale ?? 1;
|
|
239
752
|
const png = resolve(tmp, "cursor.png");
|
|
240
|
-
await renderCursorPng(
|
|
753
|
+
await renderCursorPng(
|
|
754
|
+
png,
|
|
755
|
+
Math.round(32 * pxScale * cScale),
|
|
756
|
+
cursorCfg.style ?? "arrow",
|
|
757
|
+
cursorCfg.color,
|
|
758
|
+
{
|
|
759
|
+
...(cursorCfg.outline ? { outline: cursorCfg.outline } : {}),
|
|
760
|
+
...(cursorCfg.outlineWidth != null ? { outlineWidth: cursorCfg.outlineWidth } : {}),
|
|
761
|
+
...(cursorCfg.halo ? { halo: true } : {}),
|
|
762
|
+
}
|
|
763
|
+
);
|
|
241
764
|
const withCursor = resolve(tmp, "content-cursor.mp4");
|
|
242
765
|
await runFfmpeg([
|
|
243
766
|
"-i",
|
|
@@ -272,11 +795,40 @@ export async function compose(
|
|
|
272
795
|
}
|
|
273
796
|
|
|
274
797
|
// Auto-zoom pass over the content (never over the cards or captions).
|
|
798
|
+
let focusDropped = 0;
|
|
275
799
|
if (zoomCfg && zoomEvents.length > 0) {
|
|
276
800
|
const contentMs = await probeDurationMs(content);
|
|
277
|
-
const filter = buildZoomFilter(
|
|
801
|
+
const filter = buildZoomFilter(
|
|
802
|
+
zoomEvents,
|
|
803
|
+
zoomCfg,
|
|
804
|
+
outW,
|
|
805
|
+
outH,
|
|
806
|
+
contentMs,
|
|
807
|
+
FPS,
|
|
808
|
+
storyboard.video.width,
|
|
809
|
+
(dropped, total) => {
|
|
810
|
+
focusDropped = dropped;
|
|
811
|
+
log(
|
|
812
|
+
` ! auto-zoom: ${dropped} of ${total} focus point(s) dropped to fit ` +
|
|
813
|
+
`ffmpeg's expression budget — the rest are spread evenly. Set ` +
|
|
814
|
+
`"zoom": false on scenes that don't need the camera to choose which.`
|
|
815
|
+
);
|
|
816
|
+
warn(
|
|
817
|
+
"focus-dropped",
|
|
818
|
+
`auto-zoom dropped ${dropped} of ${total} focus point(s) to fit ffmpeg's ` +
|
|
819
|
+
`expression budget — set "zoom": false on scenes that don't need the camera`
|
|
820
|
+
);
|
|
821
|
+
}
|
|
822
|
+
);
|
|
278
823
|
if (filter) {
|
|
279
|
-
|
|
824
|
+
const capped = clampScaleForWidth(zoomCfg.scale, storyboard.video.width);
|
|
825
|
+
log(
|
|
826
|
+
`auto-zoom: ${zoomEvents.length} focus event(s)` +
|
|
827
|
+
(capped < zoomCfg.scale
|
|
828
|
+
? ` (scale ${zoomCfg.scale} capped to ${capped.toFixed(2)} for a ` +
|
|
829
|
+
`${storyboard.video.width}px-wide viewport)`
|
|
830
|
+
: "")
|
|
831
|
+
);
|
|
280
832
|
const zoomed = resolve(tmp, "content-zoomed.mp4");
|
|
281
833
|
await runFfmpeg([
|
|
282
834
|
"-i",
|
|
@@ -327,19 +879,158 @@ export async function compose(
|
|
|
327
879
|
log(`motion blur: tmix ${frames} frames`);
|
|
328
880
|
}
|
|
329
881
|
|
|
882
|
+
// Keystroke chips are screen-fixed UI: after the zoom/blur, before cards.
|
|
883
|
+
let keyItems: OverlayItem[] = [];
|
|
884
|
+
if (keysPlaced.length) {
|
|
885
|
+
keyItems = await renderKeyChipPngs(
|
|
886
|
+
keysPlaced,
|
|
887
|
+
resolve(tmp, "attention"),
|
|
888
|
+
storyboard.video.width,
|
|
889
|
+
storyboard.video.height,
|
|
890
|
+
pxScale
|
|
891
|
+
);
|
|
892
|
+
const r = await overlayPngBatches(content, keyItems, tmp, "keys");
|
|
893
|
+
content = r.video;
|
|
894
|
+
log(`keystrokes: ${keysPlaced.length} chip(s) in ${r.passes} pass(es)`);
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
// Produced-look frame (opt-in `frame` block): pad the content onto a canvas
|
|
898
|
+
// and overlay the rasterized frame PNG (background, shadow, chrome, rounded
|
|
899
|
+
// hole). Everything after this — cards, watermark, captions — renders at
|
|
900
|
+
// the canvas size, so captions land in the padding band.
|
|
901
|
+
let canvasW = outW;
|
|
902
|
+
let canvasH = outH;
|
|
903
|
+
const outCfg = resolveOutputSizing(storyboard.output);
|
|
904
|
+
if (storyboard.frame) {
|
|
905
|
+
// `frame.url: "auto"` tracks the page: one frame PNG per distinct address,
|
|
906
|
+
// time-gated to the scenes that are on it (#52). Everything else renders a
|
|
907
|
+
// single static frame, exactly as before.
|
|
908
|
+
const autoUrl = storyboard.frame.url === "auto";
|
|
909
|
+
const spans = autoUrl ? frameUrlSpans(composedTl, sceneSpans) : [];
|
|
910
|
+
const labels = autoUrl ? [...new Set(spans.map((sp) => sp.label))] : [""];
|
|
911
|
+
const framePngs: string[] = [];
|
|
912
|
+
let layout!: Awaited<ReturnType<typeof renderFramePng>>;
|
|
913
|
+
for (let li = 0; li < labels.length; li++) {
|
|
914
|
+
const framePng = resolve(tmp, `frame-${li}.png`);
|
|
915
|
+
layout = await renderFramePng(
|
|
916
|
+
autoUrl ? { ...storyboard.frame, url: labels[li] || undefined } : storyboard.frame,
|
|
917
|
+
storyboard.brand,
|
|
918
|
+
framePng,
|
|
919
|
+
storyboard.video.width,
|
|
920
|
+
storyboard.video.height,
|
|
921
|
+
pxScale,
|
|
922
|
+
outCfg.width != null && outCfg.height != null ? outCfg.width / outCfg.height : undefined
|
|
923
|
+
);
|
|
924
|
+
framePngs.push(framePng);
|
|
925
|
+
}
|
|
926
|
+
const framed = resolve(tmp, "content-framed.mp4");
|
|
927
|
+
// Each frame PNG covers the whole canvas and the spans partition the
|
|
928
|
+
// timeline, so exactly one overlay is enabled at any instant.
|
|
929
|
+
const chain = framePngs
|
|
930
|
+
.map((_, li) => {
|
|
931
|
+
const win = spans
|
|
932
|
+
.filter((sp) => sp.label === labels[li])
|
|
933
|
+
.map((sp) => `between(t,${(sp.a / 1000).toFixed(3)},${(sp.b / 1000).toFixed(3)})`)
|
|
934
|
+
.join("+");
|
|
935
|
+
const gate = autoUrl && framePngs.length > 1 ? `:enable='${win}'` : "";
|
|
936
|
+
const src = li === 0 ? "[p]" : `[v${li - 1}]`;
|
|
937
|
+
const dst = li === framePngs.length - 1 ? "[vout]" : `[v${li}]`;
|
|
938
|
+
return `${src}[${li + 1}:v]overlay=0:0:format=auto${gate}${dst}`;
|
|
939
|
+
})
|
|
940
|
+
.join(";");
|
|
941
|
+
await runFfmpeg([
|
|
942
|
+
"-i",
|
|
943
|
+
content,
|
|
944
|
+
...framePngs.flatMap((f) => ["-i", f]),
|
|
945
|
+
"-filter_complex",
|
|
946
|
+
`[0:v]pad=${layout.canvasW}:${layout.canvasH}:${layout.offsetX}:${layout.offsetY}:color=${layout.padColor}[p];` +
|
|
947
|
+
chain,
|
|
948
|
+
"-map",
|
|
949
|
+
"[vout]",
|
|
950
|
+
"-an",
|
|
951
|
+
"-r",
|
|
952
|
+
String(FPS),
|
|
953
|
+
"-c:v",
|
|
954
|
+
"libx264",
|
|
955
|
+
"-preset",
|
|
956
|
+
"veryfast",
|
|
957
|
+
"-crf",
|
|
958
|
+
"20",
|
|
959
|
+
"-pix_fmt",
|
|
960
|
+
"yuv420p",
|
|
961
|
+
framed,
|
|
962
|
+
]);
|
|
963
|
+
content = framed;
|
|
964
|
+
canvasW = layout.canvasW;
|
|
965
|
+
canvasH = layout.canvasH;
|
|
966
|
+
log(
|
|
967
|
+
`frame: ${layout.canvasW}x${layout.canvasH} canvas (padding ${storyboard.frame.padding ?? 48}` +
|
|
968
|
+
`${storyboard.frame.chrome && storyboard.frame.chrome !== "none" ? `, ${storyboard.frame.chrome} chrome` : ""})` +
|
|
969
|
+
(autoUrl ? `, auto url: ${labels.filter(Boolean).join(" -> ") || "(none)"}` : "")
|
|
970
|
+
);
|
|
971
|
+
}
|
|
972
|
+
const logicalW = canvasW / pxScale;
|
|
973
|
+
const logicalH = canvasH / pxScale;
|
|
974
|
+
|
|
975
|
+
// Brand watermark (opt-in `brand.logo`): scaled logo in a corner over the
|
|
976
|
+
// content (not the cards — the cards carry the logo in their layout).
|
|
977
|
+
const logoPath = storyboard.brand?.logo
|
|
978
|
+
? resolve(project.dir, storyboard.brand.logo)
|
|
979
|
+
: null;
|
|
980
|
+
if (logoPath && storyboard.brand?.watermark?.enabled !== false) {
|
|
981
|
+
if (!(await exists(logoPath))) {
|
|
982
|
+
warn("brand-logo-missing", `brand.logo not found at ${logoPath} — watermark and card logo skipped`);
|
|
983
|
+
} else {
|
|
984
|
+
const wm = storyboard.brand?.watermark ?? {};
|
|
985
|
+
const wmW = Math.round(canvasW * (wm.scale ?? 0.11));
|
|
986
|
+
const margin = Math.round(24 * pxScale);
|
|
987
|
+
const pos = wm.position ?? "bottom-right";
|
|
988
|
+
const x = pos.endsWith("left") ? margin : canvasW - wmW - margin;
|
|
989
|
+
const yExpr = pos.startsWith("top") ? `${margin}` : `${canvasH}-h-${margin}`;
|
|
990
|
+
const marked = resolve(tmp, "content-marked.mp4");
|
|
991
|
+
await runFfmpeg([
|
|
992
|
+
"-i",
|
|
993
|
+
content,
|
|
994
|
+
"-i",
|
|
995
|
+
logoPath,
|
|
996
|
+
"-filter_complex",
|
|
997
|
+
`[1:v]scale=${wmW}:-1:flags=lanczos,format=rgba,colorchannelmixer=aa=${(wm.opacity ?? 0.85).toFixed(2)}[lg];` +
|
|
998
|
+
`[0:v][lg]overlay=${x}:${yExpr}[vout]`,
|
|
999
|
+
"-map",
|
|
1000
|
+
"[vout]",
|
|
1001
|
+
"-an",
|
|
1002
|
+
"-r",
|
|
1003
|
+
String(FPS),
|
|
1004
|
+
"-c:v",
|
|
1005
|
+
"libx264",
|
|
1006
|
+
"-preset",
|
|
1007
|
+
"veryfast",
|
|
1008
|
+
"-crf",
|
|
1009
|
+
"20",
|
|
1010
|
+
"-pix_fmt",
|
|
1011
|
+
"yuv420p",
|
|
1012
|
+
marked,
|
|
1013
|
+
]);
|
|
1014
|
+
content = marked;
|
|
1015
|
+
log(`watermark: ${storyboard.brand?.logo} at ${pos}`);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
const logoDataUri =
|
|
1019
|
+
logoPath && (await exists(logoPath)) ? await fileToDataUri(logoPath) : undefined;
|
|
1020
|
+
|
|
330
1021
|
// Intro/outro title cards. Music runs under them; narration and captions
|
|
331
1022
|
// shift right by the intro's length.
|
|
332
1023
|
const segments: string[] = [];
|
|
333
1024
|
const introMs = storyboard.intro ? storyboard.intro.durationMs : 0;
|
|
334
1025
|
if (storyboard.intro) {
|
|
335
1026
|
segments.push(
|
|
336
|
-
await cardSegment(storyboard.intro, "intro", tmp, storyboard,
|
|
1027
|
+
await cardSegment(storyboard.intro, "intro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
|
|
337
1028
|
);
|
|
338
1029
|
}
|
|
339
1030
|
segments.push(content);
|
|
340
1031
|
if (storyboard.outro) {
|
|
341
1032
|
segments.push(
|
|
342
|
-
await cardSegment(storyboard.outro, "outro", tmp, storyboard,
|
|
1033
|
+
await cardSegment(storyboard.outro, "outro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
|
|
343
1034
|
);
|
|
344
1035
|
}
|
|
345
1036
|
// Re-encode when assembling cards: the card segments carry mp4 edit lists
|
|
@@ -352,29 +1043,237 @@ export async function compose(
|
|
|
352
1043
|
? content
|
|
353
1044
|
: await concatSegments(segments, resolve(tmp, "video.mp4"), tmp, "full", true);
|
|
354
1045
|
|
|
355
|
-
|
|
1046
|
+
// Caption band collisions: any attention box / key chip that reaches into
|
|
1047
|
+
// the bottom strip flips the caption to the top for the cues it overlaps.
|
|
1048
|
+
const bandTop = canvasH - Math.round((CAPTION_BOTTOM_GAP + 80) * pxScale);
|
|
1049
|
+
const collisions = [...attentionItems, ...keyItems]
|
|
1050
|
+
.filter((it) => {
|
|
1051
|
+
// Spotlights are full-frame PNGs; use the hole instead of the canvas.
|
|
1052
|
+
return it.y + pngHeightHint(it, attentionPlaced, pxScale) > bandTop;
|
|
1053
|
+
})
|
|
1054
|
+
.map((it) => ({ a: it.a * 1000, b: it.b * 1000 }));
|
|
1055
|
+
const captionsResult = await burnCaptions(
|
|
356
1056
|
project,
|
|
357
1057
|
storyboard,
|
|
358
1058
|
fullVideo,
|
|
359
1059
|
tmp,
|
|
360
1060
|
introMs,
|
|
361
|
-
pxScale
|
|
1061
|
+
pxScale,
|
|
1062
|
+
{
|
|
1063
|
+
defaultPosition: storyboard.captions?.position ?? "bottom",
|
|
1064
|
+
windows: captionWindows,
|
|
1065
|
+
collisions,
|
|
1066
|
+
look: { ...(storyboard.captions ?? {}), font: storyboard.captions?.font ?? storyboard.brand?.font },
|
|
1067
|
+
logicalWidth: logicalW,
|
|
1068
|
+
}
|
|
362
1069
|
);
|
|
1070
|
+
const captioned = captionsResult.video;
|
|
363
1071
|
// Optional final resize/reframe (opt-in). Applied last, over the whole
|
|
364
1072
|
// composed frame (cards + captions included), so a 1280x720 take can render
|
|
365
1073
|
// as a vertical social clip. Audio is muxed after, so mux stays -c:v copy.
|
|
366
1074
|
// `output` may carry only a `loudness` override (no width/height) — resize
|
|
367
1075
|
// just when both dimensions are set.
|
|
368
1076
|
const finalVideo =
|
|
369
|
-
|
|
370
|
-
? await resizeOutput(captioned,
|
|
1077
|
+
outCfg.width != null && outCfg.height != null
|
|
1078
|
+
? await resizeOutput(captioned, outCfg, tmp)
|
|
371
1079
|
: captioned;
|
|
372
|
-
|
|
1080
|
+
const chapters = storyboard.output?.chapters
|
|
1081
|
+
? await writeChapters(tmp, [
|
|
1082
|
+
...(storyboard.intro
|
|
1083
|
+
? [{ id: "intro", title: storyboard.intro.title, durationMs: introMs }]
|
|
1084
|
+
: []),
|
|
1085
|
+
...sceneReports.map((r) => ({
|
|
1086
|
+
id: r.id,
|
|
1087
|
+
title: sceneById.get(r.id)?.title ?? r.id,
|
|
1088
|
+
durationMs: r.targetMs,
|
|
1089
|
+
})),
|
|
1090
|
+
...(storyboard.outro
|
|
1091
|
+
? [{ id: "outro", title: storyboard.outro.title, durationMs: storyboard.outro.durationMs }]
|
|
1092
|
+
: []),
|
|
1093
|
+
])
|
|
1094
|
+
: null;
|
|
1095
|
+
await muxAudio(project, storyboard, finalVideo, introMs, chapters);
|
|
373
1096
|
ok(`final video → ${project.outputPath}`);
|
|
1097
|
+
// Blank-open measurement + poster (issue #47). The poster used to be taken
|
|
1098
|
+
// at a fixed offset, which on a demo that opens with a `goto` is exactly the
|
|
1099
|
+
// white load frame; pick the first frame that actually shows content instead.
|
|
1100
|
+
const blankOpenMs = await probeFlatLeadMs(project.outputPath, 4000, introMs).catch(() => 0);
|
|
1101
|
+
if (blankOpenMs > 250) {
|
|
1102
|
+
warn(
|
|
1103
|
+
"blank-open",
|
|
1104
|
+
`the video opens on ${Math.round(blankOpenMs)}ms of blank page${storyboard.intro ? " after the intro card" : ""} ` +
|
|
1105
|
+
`(the first scene's load) — ` +
|
|
1106
|
+
`set output.trimLeadingBlank, or wait on the page's content before the first beat`
|
|
1107
|
+
);
|
|
1108
|
+
}
|
|
1109
|
+
let poster: string | undefined;
|
|
1110
|
+
if (storyboard.output?.poster) {
|
|
1111
|
+
poster = project.posterPath;
|
|
1112
|
+
const at =
|
|
1113
|
+
typeof storyboard.output.poster === "number"
|
|
1114
|
+
? storyboard.output.poster
|
|
1115
|
+
: introMs + Math.max(500, blankOpenMs + 200);
|
|
1116
|
+
await runFfmpeg([
|
|
1117
|
+
"-ss",
|
|
1118
|
+
(at / 1000).toFixed(3),
|
|
1119
|
+
"-i",
|
|
1120
|
+
project.outputPath,
|
|
1121
|
+
"-frames:v",
|
|
1122
|
+
"1",
|
|
1123
|
+
"-update",
|
|
1124
|
+
"1",
|
|
1125
|
+
poster,
|
|
1126
|
+
]);
|
|
1127
|
+
log(`poster → ${poster} (@ ${stampMs(at)})`);
|
|
1128
|
+
}
|
|
1129
|
+
// Dump the queued warning frames (issue #59). Best-effort: a failed grab
|
|
1130
|
+
// must never fail a compose that otherwise produced a video.
|
|
1131
|
+
if (warningFrames.length) {
|
|
1132
|
+
const dir = project.p("output", "warnings");
|
|
1133
|
+
await ensureDir(dir);
|
|
1134
|
+
for (const f of warningFrames) {
|
|
1135
|
+
const png = resolve(dir, `${f.name}.png`);
|
|
1136
|
+
await runFfmpeg([
|
|
1137
|
+
"-y",
|
|
1138
|
+
"-ss",
|
|
1139
|
+
((introMs + f.atMs) / 1000).toFixed(3),
|
|
1140
|
+
"-i",
|
|
1141
|
+
project.outputPath,
|
|
1142
|
+
"-frames:v",
|
|
1143
|
+
"1",
|
|
1144
|
+
"-update",
|
|
1145
|
+
"1",
|
|
1146
|
+
png,
|
|
1147
|
+
]).catch(() => {});
|
|
1148
|
+
}
|
|
1149
|
+
log(`warning frames → ${relative(project.dir, dir)}/ (${warningFrames.length})`);
|
|
1150
|
+
}
|
|
374
1151
|
// AIDEMO_KEEP_TMP=1 keeps .compose-tmp for debugging intermediates.
|
|
375
1152
|
if (!process.env.AIDEMO_KEEP_TMP) {
|
|
376
1153
|
await fs.rm(tmp, { recursive: true, force: true });
|
|
377
1154
|
}
|
|
1155
|
+
|
|
1156
|
+
const report: ComposeReport = {
|
|
1157
|
+
output: project.outputPath,
|
|
1158
|
+
durationMs: await probeDurationMs(project.outputPath).catch(() => 0),
|
|
1159
|
+
scenes: sceneReports,
|
|
1160
|
+
zoom: { focusTotal: zoomEvents.length, focusDropped },
|
|
1161
|
+
cursorPoints: cursorPts.length,
|
|
1162
|
+
captions: { cues: captionsResult.cues, passes: captionsResult.passes },
|
|
1163
|
+
hold: hold.mode,
|
|
1164
|
+
...(poster ? { poster } : {}),
|
|
1165
|
+
blankOpenMs: Math.round(blankOpenMs),
|
|
1166
|
+
...(attentionPlaced.length || keysPlaced.length || clicksPlaced.length || redactPlaced.length
|
|
1167
|
+
? {
|
|
1168
|
+
attention: {
|
|
1169
|
+
events: attentionPlaced.length,
|
|
1170
|
+
keys: keysPlaced.length,
|
|
1171
|
+
clicks: clicksPlaced.length,
|
|
1172
|
+
redactSpans: redactPlaced.length,
|
|
1173
|
+
captionsFlipped: captionsResult.flipped,
|
|
1174
|
+
},
|
|
1175
|
+
}
|
|
1176
|
+
: {}),
|
|
1177
|
+
warnings,
|
|
1178
|
+
};
|
|
1179
|
+
await writeJson(project.reportPath, report);
|
|
1180
|
+
const frozen = sceneReports.filter((s) => s.holdPct > FREEZE_WARN_PCT).length;
|
|
1181
|
+
log(
|
|
1182
|
+
`report → ${project.reportPath}` +
|
|
1183
|
+
(frozen ? ` (${frozen} of ${sceneReports.length} scene(s) mostly held)` : "")
|
|
1184
|
+
);
|
|
1185
|
+
if (warnings.length) {
|
|
1186
|
+
// Last thing on screen, grouped by scene (issue #59): in a 28-scene render
|
|
1187
|
+
// an ungrouped list scrolls away above the ✓ lines and is never read.
|
|
1188
|
+
log(`⚠ ${warnings.length} compose warning(s):`);
|
|
1189
|
+
const byScene = new Map<string, ComposeWarning[]>();
|
|
1190
|
+
for (const w of warnings) {
|
|
1191
|
+
const k = w.scene ?? "(whole video)";
|
|
1192
|
+
byScene.set(k, [...(byScene.get(k) ?? []), w]);
|
|
1193
|
+
}
|
|
1194
|
+
for (const [scene, list] of byScene) {
|
|
1195
|
+
log(` ${scene}:`);
|
|
1196
|
+
for (const w of list) log(` - [${w.code}] ${w.message}`);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
return report;
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
/** Share of a scene that may be a held frame before compose flags it. */
|
|
1203
|
+
const FREEZE_WARN_PCT = 0.4;
|
|
1204
|
+
|
|
1205
|
+
/**
|
|
1206
|
+
* Hold mode "drift": instead of cloning the scene's last frame for `holdMs`,
|
|
1207
|
+
* push in on it very slowly (1 → driftScale over the hold), so a long
|
|
1208
|
+
* narration over a static page reads as a deliberate dwell, not a stall.
|
|
1209
|
+
* The frame is pulled from the encoded scene's tail, rendered as a zoompan
|
|
1210
|
+
* still (pre-upscaled 2x so integer-pixel crops don't shimmer — the same rule
|
|
1211
|
+
* the auto-zoom pass follows), and re-encoded onto the scene. One tiny
|
|
1212
|
+
* expression, so the zoom budget is untouched; `zoompan`/`scale` are core.
|
|
1213
|
+
*/
|
|
1214
|
+
async function appendDriftHold(
|
|
1215
|
+
scenePath: string,
|
|
1216
|
+
holdMs: number,
|
|
1217
|
+
driftScale: number,
|
|
1218
|
+
outW: number,
|
|
1219
|
+
outH: number,
|
|
1220
|
+
tmp: string,
|
|
1221
|
+
sceneIdx: number
|
|
1222
|
+
): Promise<string> {
|
|
1223
|
+
const durMs = await probeDurationMs(scenePath);
|
|
1224
|
+
const framePng = resolve(tmp, `scene-${sceneIdx}-hold.png`);
|
|
1225
|
+
await runFfmpeg([
|
|
1226
|
+
"-ss",
|
|
1227
|
+
(Math.max(0, durMs - 60) / 1000).toFixed(3),
|
|
1228
|
+
"-i",
|
|
1229
|
+
scenePath,
|
|
1230
|
+
"-frames:v",
|
|
1231
|
+
"1",
|
|
1232
|
+
"-update",
|
|
1233
|
+
"1",
|
|
1234
|
+
framePng,
|
|
1235
|
+
]);
|
|
1236
|
+
const holdSec = holdMs / 1000;
|
|
1237
|
+
const frames = Math.max(1, Math.round(holdSec * FPS));
|
|
1238
|
+
const holdMp4 = resolve(tmp, `scene-${sceneIdx}-hold.mp4`);
|
|
1239
|
+
// Ease the creep (ease-out quad) so it starts moving right away and settles.
|
|
1240
|
+
const z = `'1+(${driftScale.toFixed(4)}-1)*(1-pow(1-min(on/${frames}\,1)\,2))'`;
|
|
1241
|
+
await runFfmpeg([
|
|
1242
|
+
"-loop",
|
|
1243
|
+
"1",
|
|
1244
|
+
"-framerate",
|
|
1245
|
+
String(FPS),
|
|
1246
|
+
"-i",
|
|
1247
|
+
framePng,
|
|
1248
|
+
"-t",
|
|
1249
|
+
holdSec.toFixed(3),
|
|
1250
|
+
"-vf",
|
|
1251
|
+
`scale=iw*2:ih*2:flags=lanczos,` +
|
|
1252
|
+
`zoompan=z=${z}:x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${outW}x${outH}:fps=${FPS},` +
|
|
1253
|
+
`setsar=1`,
|
|
1254
|
+
"-r",
|
|
1255
|
+
String(FPS),
|
|
1256
|
+
"-an",
|
|
1257
|
+
"-c:v",
|
|
1258
|
+
"libx264",
|
|
1259
|
+
"-preset",
|
|
1260
|
+
"veryfast",
|
|
1261
|
+
"-crf",
|
|
1262
|
+
"20",
|
|
1263
|
+
"-pix_fmt",
|
|
1264
|
+
"yuv420p",
|
|
1265
|
+
holdMp4,
|
|
1266
|
+
]);
|
|
1267
|
+
// The PNG-sourced hold carries an edit list; re-encode the join so the pts
|
|
1268
|
+
// start clean at 0 (a stream-copy concat would leak negative timestamps
|
|
1269
|
+
// into the overlay passes — see AGENTS.md).
|
|
1270
|
+
return concatSegments(
|
|
1271
|
+
[scenePath, holdMp4],
|
|
1272
|
+
resolve(tmp, `scene-${sceneIdx}-held.mp4`),
|
|
1273
|
+
tmp,
|
|
1274
|
+
`scene-${sceneIdx}-held`,
|
|
1275
|
+
true
|
|
1276
|
+
);
|
|
378
1277
|
}
|
|
379
1278
|
|
|
380
1279
|
/**
|
|
@@ -384,7 +1283,7 @@ export async function compose(
|
|
|
384
1283
|
* needs an API key so compose can't regenerate them itself; warn loudly and
|
|
385
1284
|
* name the fix instead.
|
|
386
1285
|
*/
|
|
387
|
-
async function warnStaleCaptions(project: Project): Promise<
|
|
1286
|
+
async function warnStaleCaptions(project: Project): Promise<boolean> {
|
|
388
1287
|
try {
|
|
389
1288
|
const [cap, narr] = await Promise.all([
|
|
390
1289
|
fs.stat(project.captionsCuesPath),
|
|
@@ -396,10 +1295,12 @@ async function warnStaleCaptions(project: Project): Promise<void> {
|
|
|
396
1295
|
`the burned captions will not match the audio. ` +
|
|
397
1296
|
`Fix: aidemo captions ${project.dir}` + ` (then re-run compose).`
|
|
398
1297
|
);
|
|
1298
|
+
return true;
|
|
399
1299
|
}
|
|
400
1300
|
} catch {
|
|
401
1301
|
/* captions or narration absent — burnCaptions handles missing captions */
|
|
402
1302
|
}
|
|
1303
|
+
return false;
|
|
403
1304
|
}
|
|
404
1305
|
|
|
405
1306
|
/**
|
|
@@ -450,12 +1351,96 @@ async function concatSegments(
|
|
|
450
1351
|
* padded). Net effect: identical A/V duration + alignment as hard cuts, but the
|
|
451
1352
|
* boundaries cross-dissolve. Re-encodes (xfade needs filter_complex).
|
|
452
1353
|
*/
|
|
1354
|
+
/**
|
|
1355
|
+
* Does the boundary between these two scenes cross a page navigation? Either
|
|
1356
|
+
* the next scene opens with a load (its first idle span is the `goto` wait) or
|
|
1357
|
+
* the previous scene's last action was a navigation. Both are already in the
|
|
1358
|
+
* timeline, so no extra probing is needed.
|
|
1359
|
+
*/
|
|
1360
|
+
/**
|
|
1361
|
+
* Per-scene address-pill text for `frame.url: "auto"`: the host of the URL the
|
|
1362
|
+
* scene is on, carried forward from the last `goto` (falling back to
|
|
1363
|
+
* `baseUrl`). Consecutive scenes on the same host share one span.
|
|
1364
|
+
*/
|
|
1365
|
+
function frameUrlSpans(
|
|
1366
|
+
scenes: Timeline["scenes"],
|
|
1367
|
+
spans: Array<{ a: number; b: number }>
|
|
1368
|
+
): Array<{ a: number; b: number; label: string }> {
|
|
1369
|
+
let current = "";
|
|
1370
|
+
const out: Array<{ a: number; b: number; label: string }> = [];
|
|
1371
|
+
for (let i = 0; i < scenes.length; i++) {
|
|
1372
|
+
for (const a of scenes[i].actions ?? []) {
|
|
1373
|
+
if (a.op === "goto" && a.target) current = prettyHost(a.target) || current;
|
|
1374
|
+
}
|
|
1375
|
+
const span = spans[i];
|
|
1376
|
+
if (!span) continue;
|
|
1377
|
+
const last = out.at(-1);
|
|
1378
|
+
if (last && last.label === current) last.b = span.b;
|
|
1379
|
+
else out.push({ a: i === 0 ? 0 : span.a, b: span.b, label: current });
|
|
1380
|
+
}
|
|
1381
|
+
if (out.length) out[out.length - 1].b += 3600_000; // cover holds/cards past the last scene
|
|
1382
|
+
return out;
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
function prettyHost(url?: string): string {
|
|
1386
|
+
if (!url) return "";
|
|
1387
|
+
try {
|
|
1388
|
+
const u = new URL(url, "http://localhost");
|
|
1389
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") return "";
|
|
1390
|
+
return u.host.replace(/^www\./, "") + (u.pathname !== "/" ? u.pathname : "");
|
|
1391
|
+
} catch {
|
|
1392
|
+
return "";
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
function crossesNavigation(
|
|
1397
|
+
prev: Timeline["scenes"][number],
|
|
1398
|
+
next: Timeline["scenes"][number]
|
|
1399
|
+
): boolean {
|
|
1400
|
+
const opensOnLoad = (next.idleSpans ?? []).some(
|
|
1401
|
+
(sp) => sp.label === "load" && sp.startMs - next.startMs < 1500
|
|
1402
|
+
);
|
|
1403
|
+
if (opensOnLoad) return true;
|
|
1404
|
+
const lastOp = (prev.actions ?? []).filter((a) => !a.skipped).at(-1)?.op;
|
|
1405
|
+
const firstOp = (next.actions ?? [])[0]?.op;
|
|
1406
|
+
return lastOp === "goto" || lastOp === "back" || firstOp === "goto" || firstOp === "back";
|
|
1407
|
+
}
|
|
1408
|
+
|
|
453
1409
|
async function crossfadeScenes(
|
|
454
1410
|
sceneVideos: string[],
|
|
455
|
-
|
|
456
|
-
outPath: string
|
|
1411
|
+
durationsMs: number[],
|
|
1412
|
+
outPath: string,
|
|
1413
|
+
tmp: string
|
|
457
1414
|
): Promise<string> {
|
|
458
|
-
|
|
1415
|
+
// Hard-cut boundaries split the run into groups: each group is crossfaded on
|
|
1416
|
+
// its own (duration is preserved inside a group) and the groups are then
|
|
1417
|
+
// concatenated, which IS the hard cut.
|
|
1418
|
+
if (durationsMs.some((d) => d <= 0)) {
|
|
1419
|
+
const groups: string[][] = [[sceneVideos[0]]];
|
|
1420
|
+
const groupDurs: number[][] = [[]];
|
|
1421
|
+
for (let i = 1; i < sceneVideos.length; i++) {
|
|
1422
|
+
if (durationsMs[i - 1] <= 0) {
|
|
1423
|
+
groups.push([sceneVideos[i]]);
|
|
1424
|
+
groupDurs.push([]);
|
|
1425
|
+
} else {
|
|
1426
|
+
groups[groups.length - 1].push(sceneVideos[i]);
|
|
1427
|
+
groupDurs[groupDurs.length - 1].push(durationsMs[i - 1]);
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
const parts: string[] = [];
|
|
1431
|
+
for (let g = 0; g < groups.length; g++) {
|
|
1432
|
+
if (groups[g].length === 1) {
|
|
1433
|
+
parts.push(groups[g][0]);
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
parts.push(
|
|
1437
|
+
await crossfadeScenes(groups[g], groupDurs[g], resolve(tmp, `xfade-${g}.mp4`), tmp)
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
if (parts.length === 1) return parts[0];
|
|
1441
|
+
return concatSegments(parts, outPath, tmp, "xfade-groups", true);
|
|
1442
|
+
}
|
|
1443
|
+
const D = Math.max(0.05, Math.max(...durationsMs) / 1000);
|
|
459
1444
|
// Measure each segment so the offsets track real (not assumed) durations.
|
|
460
1445
|
const durs = await Promise.all(sceneVideos.map((p) => probeDurationMs(p)));
|
|
461
1446
|
// Freeze-tail margin: a touch longer than D so the fade never starves at the
|
|
@@ -484,8 +1469,9 @@ async function crossfadeScenes(
|
|
|
484
1469
|
cumMs += durs[i - 1];
|
|
485
1470
|
const offset = (cumMs / 1000).toFixed(3);
|
|
486
1471
|
const out = i === sceneVideos.length - 1 ? "xout" : `x${i}`;
|
|
1472
|
+
const d = Math.max(0.05, durationsMs[i - 1] / 1000);
|
|
487
1473
|
filters.push(
|
|
488
|
-
`[${prev}][${labels[i]}]xfade=transition=fade:duration=${
|
|
1474
|
+
`[${prev}][${labels[i]}]xfade=transition=fade:duration=${d.toFixed(
|
|
489
1475
|
3
|
|
490
1476
|
)}:offset=${offset}[${out}]`
|
|
491
1477
|
);
|
|
@@ -564,18 +1550,23 @@ async function cardSegment(
|
|
|
564
1550
|
name: string,
|
|
565
1551
|
tmp: string,
|
|
566
1552
|
storyboard: Storyboard,
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
pxScale: number
|
|
1553
|
+
logicalW: number,
|
|
1554
|
+
logicalH: number,
|
|
1555
|
+
pxScale: number,
|
|
1556
|
+
logoDataUri?: string
|
|
570
1557
|
): Promise<string> {
|
|
571
1558
|
const png = resolve(tmp, `${name}.png`);
|
|
572
1559
|
await renderCardPng(
|
|
573
1560
|
card,
|
|
574
1561
|
png,
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
pxScale
|
|
1562
|
+
Math.round(logicalW),
|
|
1563
|
+
Math.round(logicalH),
|
|
1564
|
+
pxScale,
|
|
1565
|
+
storyboard.brand,
|
|
1566
|
+
logoDataUri
|
|
578
1567
|
);
|
|
1568
|
+
const outW = Math.round(logicalW * pxScale) & ~1;
|
|
1569
|
+
const outH = Math.round(logicalH * pxScale) & ~1;
|
|
579
1570
|
const durSec = card.durationMs / 1000;
|
|
580
1571
|
const fadeSec = Math.min(card.fadeMs / 1000, durSec / 3);
|
|
581
1572
|
const out = resolve(tmp, `${name}.mp4`);
|
|
@@ -609,6 +1600,60 @@ async function cardSegment(
|
|
|
609
1600
|
return out;
|
|
610
1601
|
}
|
|
611
1602
|
|
|
1603
|
+
/** Union of overlapping/adjacent idle spans (auto + annotated), sorted. */
|
|
1604
|
+
function mergeIdle(spans: IdleSpan[]): IdleSpan[] {
|
|
1605
|
+
const sorted = [...spans].sort((a, b) => a.startMs - b.startMs);
|
|
1606
|
+
const out: IdleSpan[] = [];
|
|
1607
|
+
for (const s of sorted) {
|
|
1608
|
+
const last = out[out.length - 1];
|
|
1609
|
+
if (last && s.startMs <= last.endMs) last.endMs = Math.max(last.endMs, s.endMs);
|
|
1610
|
+
else out.push({ ...s });
|
|
1611
|
+
}
|
|
1612
|
+
return out;
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
/**
|
|
1616
|
+
* Motionless spans of the raw file (video time) → idle spans for one scene
|
|
1617
|
+
* (timeline time, the same base the player writes). Spans are clipped to the
|
|
1618
|
+
* scene, and the first/last 300 ms are left alone: the frame right after a
|
|
1619
|
+
* click and the one the scene ends on are the beats a viewer reads, and
|
|
1620
|
+
* trimming into them makes cuts feel clipped.
|
|
1621
|
+
*/
|
|
1622
|
+
function sceneAutoIdle(
|
|
1623
|
+
freezes: FreezeSpan[],
|
|
1624
|
+
tl: Timeline["scenes"][number],
|
|
1625
|
+
leadInMs: number
|
|
1626
|
+
): IdleSpan[] {
|
|
1627
|
+
const EDGE_MS = 300;
|
|
1628
|
+
const lo = tl.startMs + leadInMs + EDGE_MS;
|
|
1629
|
+
const hi = tl.endMs + leadInMs - EDGE_MS;
|
|
1630
|
+
// Windows the author explicitly asked to be ON SCREEN: an attention overlay
|
|
1631
|
+
// and its hold, a deliberate `focus`, a named still. A static confirmation
|
|
1632
|
+
// card is motionless by nature, so freezedetect happily proposes trimming
|
|
1633
|
+
// exactly the beat the scene exists for — and compose then holds the NEXT
|
|
1634
|
+
// page's frame under that scene's narration (issue #55).
|
|
1635
|
+
const protectedWindows: Array<[number, number]> = [
|
|
1636
|
+
...(tl.attentionEvents ?? []).map(
|
|
1637
|
+
(e): [number, number] => [e.tMs + leadInMs, e.tMs + e.holdMs + leadInMs]
|
|
1638
|
+
),
|
|
1639
|
+
...(tl.focusEvents ?? [])
|
|
1640
|
+
.filter((e) => e.kind === "focus")
|
|
1641
|
+
.map((e): [number, number] => [e.tMs + leadInMs, e.tMs + (e.holdMs ?? 1200) + leadInMs]),
|
|
1642
|
+
...(tl.stillEvents ?? []).map(
|
|
1643
|
+
(e): [number, number] => [e.tMs + leadInMs - 200, e.tMs + leadInMs + 200]
|
|
1644
|
+
),
|
|
1645
|
+
];
|
|
1646
|
+
const out: IdleSpan[] = [];
|
|
1647
|
+
for (const f of freezes) {
|
|
1648
|
+
const a = Math.max(f.startMs, lo);
|
|
1649
|
+
const b = Math.min(f.endMs, hi);
|
|
1650
|
+
if (b - a <= 200) continue;
|
|
1651
|
+
if (protectedWindows.some(([pa, pb]) => pa < b && pb > a)) continue;
|
|
1652
|
+
out.push({ startMs: a - leadInMs, endMs: b - leadInMs, label: "auto-idle" });
|
|
1653
|
+
}
|
|
1654
|
+
return out;
|
|
1655
|
+
}
|
|
1656
|
+
|
|
612
1657
|
/**
|
|
613
1658
|
* Build the list of [startMs,endMs] video-time spans to keep for a scene:
|
|
614
1659
|
* non-idle content in full, each idle span capped to IDLE_CAP_MS.
|
|
@@ -641,6 +1686,14 @@ function keepIntervals(
|
|
|
641
1686
|
return keeps.filter(([a, b]) => b - a > 20);
|
|
642
1687
|
}
|
|
643
1688
|
|
|
1689
|
+
/** One retime piece: kept-span offsets [a,b) play at factor f, starting at stretched ms `out`. */
|
|
1690
|
+
interface RetimePiece {
|
|
1691
|
+
a: number;
|
|
1692
|
+
b: number;
|
|
1693
|
+
f: number;
|
|
1694
|
+
out: number;
|
|
1695
|
+
}
|
|
1696
|
+
|
|
644
1697
|
/**
|
|
645
1698
|
* Where a raw-video timestamp lands inside the concatenation of the kept
|
|
646
1699
|
* spans. Timestamps inside a trimmed gap clamp to the gap's start.
|
|
@@ -655,6 +1708,24 @@ function offsetInKeeps(keeps: Array<[number, number]>, t: number): number {
|
|
|
655
1708
|
return off;
|
|
656
1709
|
}
|
|
657
1710
|
|
|
1711
|
+
/** Inverse of `offsetInKeeps`: kept-span offset → raw video ms. */
|
|
1712
|
+
function rawAtKeepOffset(keeps: Array<[number, number]>, local: number): number {
|
|
1713
|
+
let off = 0;
|
|
1714
|
+
for (const [a, b] of keeps) {
|
|
1715
|
+
const len = b - a;
|
|
1716
|
+
if (local <= off + len) return a + (local - off);
|
|
1717
|
+
off += len;
|
|
1718
|
+
}
|
|
1719
|
+
return keeps.length ? keeps[keeps.length - 1][1] : 0;
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
/** mm:ss.t for a ms offset, for log lines that point back into the raw take. */
|
|
1723
|
+
function stampMs(ms: number): string {
|
|
1724
|
+
const s = Math.max(0, ms) / 1000;
|
|
1725
|
+
const m = Math.floor(s / 60);
|
|
1726
|
+
return `${m}:${(s - m * 60).toFixed(1).padStart(4, "0")}`;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
658
1729
|
/** Extract each keep span from raw and concat into one mp4 (uniform h264). */
|
|
659
1730
|
async function extractAndConcat(
|
|
660
1731
|
rawVideo: string,
|
|
@@ -723,67 +1794,159 @@ async function extractAndConcat(
|
|
|
723
1794
|
*/
|
|
724
1795
|
const CAPTION_BOTTOM_GAP = 96;
|
|
725
1796
|
|
|
1797
|
+
/**
|
|
1798
|
+
* Max caption cues overlaid in a single ffmpeg pass. Each cue costs one input
|
|
1799
|
+
* and one filter link; ffmpeg fails to configure the graph somewhere past ~90
|
|
1800
|
+
* (issue #36), so long narrations are burned in several passes.
|
|
1801
|
+
*/
|
|
1802
|
+
const CAPTION_BATCH = 32;
|
|
1803
|
+
|
|
1804
|
+
/**
|
|
1805
|
+
* Overlay many time-gated PNGs onto `src` in passes of ≤ CAPTION_BATCH. Each
|
|
1806
|
+
* item is one extra input plus one overlay link; ffmpeg fails to configure a
|
|
1807
|
+
* graph somewhere past ~90 of them (issue #36), so long lists are burned in
|
|
1808
|
+
* several re-encoding passes. Shared by captions, attention beats, key chips.
|
|
1809
|
+
*/
|
|
1810
|
+
async function overlayPngBatches(
|
|
1811
|
+
src: string,
|
|
1812
|
+
items: OverlayItem[],
|
|
1813
|
+
tmp: string,
|
|
1814
|
+
label: string
|
|
1815
|
+
): Promise<{ video: string; passes: number }> {
|
|
1816
|
+
if (items.length === 0) return { video: src, passes: 0 };
|
|
1817
|
+
const batches: OverlayItem[][] = [];
|
|
1818
|
+
for (let i = 0; i < items.length; i += CAPTION_BATCH) {
|
|
1819
|
+
batches.push(items.slice(i, i + CAPTION_BATCH));
|
|
1820
|
+
}
|
|
1821
|
+
if (batches.length > 1) {
|
|
1822
|
+
log(`${label} overlay: ${batches.length} pass(es) of <= ${CAPTION_BATCH} item(s)`);
|
|
1823
|
+
}
|
|
1824
|
+
let cur = src;
|
|
1825
|
+
let out = src;
|
|
1826
|
+
for (let b = 0; b < batches.length; b++) {
|
|
1827
|
+
const batch = batches[b];
|
|
1828
|
+
const args: string[] = ["-i", cur];
|
|
1829
|
+
for (const it of batch) args.push("-i", it.png);
|
|
1830
|
+
const chain: string[] = [];
|
|
1831
|
+
let prev = "0:v";
|
|
1832
|
+
batch.forEach((it, i) => {
|
|
1833
|
+
const inp = `${i + 1}:v`;
|
|
1834
|
+
const lbl = i === batch.length - 1 ? "vout" : `o${i}`;
|
|
1835
|
+
// Escape the enable-expression commas so ffmpeg doesn't read them as
|
|
1836
|
+
// filter separators.
|
|
1837
|
+
chain.push(
|
|
1838
|
+
`[${prev}][${inp}]overlay=${it.x}:${it.y}:enable=between(t\\,${it.a.toFixed(3)}\\,${it.b.toFixed(3)})[${lbl}]`
|
|
1839
|
+
);
|
|
1840
|
+
prev = lbl;
|
|
1841
|
+
});
|
|
1842
|
+
out = resolve(tmp, `${label}-${b}.mp4`);
|
|
1843
|
+
await runFfmpeg([
|
|
1844
|
+
...args,
|
|
1845
|
+
"-filter_complex",
|
|
1846
|
+
chain.join(";"),
|
|
1847
|
+
"-map",
|
|
1848
|
+
"[vout]",
|
|
1849
|
+
"-an",
|
|
1850
|
+
"-c:v",
|
|
1851
|
+
"libx264",
|
|
1852
|
+
"-preset",
|
|
1853
|
+
"veryfast",
|
|
1854
|
+
"-crf",
|
|
1855
|
+
"20",
|
|
1856
|
+
"-pix_fmt",
|
|
1857
|
+
"yuv420p",
|
|
1858
|
+
out,
|
|
1859
|
+
]);
|
|
1860
|
+
cur = out;
|
|
1861
|
+
}
|
|
1862
|
+
return { video: out, passes: batches.length };
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1865
|
+
/** Height of an overlay PNG for the caption-collision check (spotlight → its hole). */
|
|
1866
|
+
function pngHeightHint(it: OverlayItem, placed: PlacedAttention[], pxScale: number): number {
|
|
1867
|
+
if (it.x === 0 && it.y === 0) {
|
|
1868
|
+
const ev = placed.find((p) => p.kind === "spotlight" && p.a === it.a && p.b === it.b);
|
|
1869
|
+
if (ev) return Math.round((ev.y + ev.h + (ev.padding ?? 10)) * pxScale);
|
|
1870
|
+
return 0;
|
|
1871
|
+
}
|
|
1872
|
+
return it.png.includes("/key-") ? 60 * pxScale : 48 * pxScale;
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
interface CaptionPlacement {
|
|
1876
|
+
defaultPosition: "top" | "bottom";
|
|
1877
|
+
/** Caption look (style/font/size/color/background), brand font already merged. */
|
|
1878
|
+
look?: CaptionsConfig & { font?: string };
|
|
1879
|
+
/** Logical (CSS px) width of the frame the strip must span. */
|
|
1880
|
+
logicalWidth?: number;
|
|
1881
|
+
/** Content-time windows (ms, before the intro shift) with a fixed position. */
|
|
1882
|
+
windows: Array<{ a: number; b: number; position: "top" | "bottom" }>;
|
|
1883
|
+
/** Content-time windows (ms) during which the bottom band is occupied. */
|
|
1884
|
+
collisions: Array<{ a: number; b: number }>;
|
|
1885
|
+
}
|
|
1886
|
+
|
|
726
1887
|
async function burnCaptions(
|
|
727
1888
|
project: Project,
|
|
728
1889
|
storyboard: Storyboard,
|
|
729
1890
|
silentVideo: string,
|
|
730
1891
|
tmp: string,
|
|
731
1892
|
introMs: number,
|
|
732
|
-
pxScale: number
|
|
733
|
-
|
|
734
|
-
|
|
1893
|
+
pxScale: number,
|
|
1894
|
+
placement: CaptionPlacement = { defaultPosition: "bottom", windows: [], collisions: [] }
|
|
1895
|
+
): Promise<{ video: string; cues: number; passes: number; flipped: number }> {
|
|
1896
|
+
const none = { video: silentVideo, cues: 0, passes: 0, flipped: 0 };
|
|
1897
|
+
if (placement.look?.style === "none") {
|
|
1898
|
+
log("captions: style none — not burned (SRT/VTT files still written)");
|
|
1899
|
+
return none;
|
|
1900
|
+
}
|
|
1901
|
+
if (!(await exists(project.captionsCuesPath))) return none;
|
|
735
1902
|
const cues = (await readJson<Cue[]>(project.captionsCuesPath)) ?? [];
|
|
736
|
-
if (cues.length === 0) return
|
|
1903
|
+
if (cues.length === 0) return none;
|
|
737
1904
|
|
|
738
1905
|
log(`rendering ${cues.length} caption image(s)`);
|
|
739
1906
|
const rendered = await renderCaptionPngs(
|
|
740
1907
|
cues,
|
|
741
1908
|
resolve(tmp, "captions"),
|
|
742
|
-
storyboard.video.width,
|
|
743
|
-
pxScale
|
|
1909
|
+
Math.round(placement.logicalWidth ?? storyboard.video.width),
|
|
1910
|
+
pxScale,
|
|
1911
|
+
placement.look ?? {}
|
|
744
1912
|
);
|
|
745
1913
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
//
|
|
750
|
-
//
|
|
751
|
-
const
|
|
752
|
-
const
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
const
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
);
|
|
765
|
-
|
|
1914
|
+
// Cue times are narration-relative; the intro card shifts them right. The
|
|
1915
|
+
// strip sits `gap` above the bottom edge (clear of an app's bottom input
|
|
1916
|
+
// bar) — or, per scene / on a collision with an attention overlay, at the
|
|
1917
|
+
// top: the PNG is a bottom-anchored strip, so "top" flips it vertically.
|
|
1918
|
+
// A "bar" strip is flush with the frame edge (no gap, no pill offset).
|
|
1919
|
+
const bar = placement.look?.style === "bar";
|
|
1920
|
+
const gap = bar ? 0 : Math.round(CAPTION_BOTTOM_GAP * pxScale);
|
|
1921
|
+
const { height: outH } = await probeVideoDims(silentVideo);
|
|
1922
|
+
let flipped = 0;
|
|
1923
|
+
const overlap = (a: number, b: number, w: { a: number; b: number }) => a < w.b && b > w.a;
|
|
1924
|
+
const items: OverlayItem[] = rendered.map((r) => {
|
|
1925
|
+
let pos = placement.defaultPosition;
|
|
1926
|
+
const win = placement.windows.find((w) => r.startMs >= w.a && r.startMs < w.b);
|
|
1927
|
+
if (win) pos = win.position;
|
|
1928
|
+
else if (pos === "bottom" && placement.collisions.some((c) => overlap(r.startMs, r.endMs, c))) {
|
|
1929
|
+
pos = "top";
|
|
1930
|
+
flipped++;
|
|
1931
|
+
}
|
|
1932
|
+
const stripH = Math.round(r.height * pxScale);
|
|
1933
|
+
// The strip PNG is bottom-anchored (pill bottom ≈ 22 px above the strip's
|
|
1934
|
+
// edge). For "top", slide the strip up so the pill's bottom lands ~gap+70
|
|
1935
|
+
// px from the top edge — a one- or two-line pill stays fully in frame.
|
|
1936
|
+
const topY = bar
|
|
1937
|
+
? -(stripH - Math.round(62 * pxScale))
|
|
1938
|
+
: gap + Math.round(70 * pxScale) - (stripH - Math.round(22 * pxScale));
|
|
1939
|
+
return {
|
|
1940
|
+
png: r.png,
|
|
1941
|
+
x: 0,
|
|
1942
|
+
y: pos === "top" ? topY : outH - stripH - gap,
|
|
1943
|
+
a: (r.startMs + introMs) / 1000,
|
|
1944
|
+
b: (r.endMs + introMs) / 1000,
|
|
1945
|
+
};
|
|
766
1946
|
});
|
|
767
|
-
|
|
768
|
-
const
|
|
769
|
-
|
|
770
|
-
...args,
|
|
771
|
-
"-filter_complex",
|
|
772
|
-
chain.join(";"),
|
|
773
|
-
"-map",
|
|
774
|
-
"[vout]",
|
|
775
|
-
"-an",
|
|
776
|
-
"-c:v",
|
|
777
|
-
"libx264",
|
|
778
|
-
"-preset",
|
|
779
|
-
"veryfast",
|
|
780
|
-
"-crf",
|
|
781
|
-
"20",
|
|
782
|
-
"-pix_fmt",
|
|
783
|
-
"yuv420p",
|
|
784
|
-
out,
|
|
785
|
-
]);
|
|
786
|
-
return out;
|
|
1947
|
+
if (flipped) log(`captions: ${flipped} cue(s) moved to the top (overlay in the bottom band)`);
|
|
1948
|
+
const r = await overlayPngBatches(silentVideo, items, tmp, "captioned");
|
|
1949
|
+
return { video: r.video, cues: cues.length, passes: r.passes, flipped };
|
|
787
1950
|
}
|
|
788
1951
|
|
|
789
1952
|
/**
|
|
@@ -795,11 +1958,54 @@ async function burnCaptions(
|
|
|
795
1958
|
* fixed -22dB bed). The bed is trimmed to the video, faded out at the end,
|
|
796
1959
|
* and the narration is delayed past the intro card.
|
|
797
1960
|
*/
|
|
1961
|
+
/** Output sizing with a preset's defaults filled in (explicit keys win). */
|
|
1962
|
+
function resolveOutputSizing(output: Output | undefined): Output {
|
|
1963
|
+
if (!output) return {};
|
|
1964
|
+
const p = output.preset ? OUTPUT_PRESETS[output.preset] : null;
|
|
1965
|
+
return {
|
|
1966
|
+
...output,
|
|
1967
|
+
width: output.width ?? p?.width,
|
|
1968
|
+
height: output.height ?? p?.height,
|
|
1969
|
+
// An explicit fit wins over the preset's (issue #58): `preset: "youtube"`
|
|
1970
|
+
// + `fit: "cover"` is how you get an edge-to-edge 16:9 cut from a 16:10
|
|
1971
|
+
// capture, and before this the preset silently forced "contain".
|
|
1972
|
+
fit: output.fit ?? p?.fit ?? "contain",
|
|
1973
|
+
};
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/** Write an ffmetadata file with one chapter per scene (content time + intro). */
|
|
1977
|
+
async function writeChapters(
|
|
1978
|
+
tmp: string,
|
|
1979
|
+
scenes: Array<{ id: string; title: string; durationMs: number }>
|
|
1980
|
+
): Promise<string> {
|
|
1981
|
+
const lines = [";FFMETADATA1"];
|
|
1982
|
+
let t = 0;
|
|
1983
|
+
for (const s of scenes) {
|
|
1984
|
+
const a = Math.round(t);
|
|
1985
|
+
const b = Math.round(t + s.durationMs);
|
|
1986
|
+
lines.push("[CHAPTER]", "TIMEBASE=1/1000", `START=${a}`, `END=${b}`, `title=${s.title.replace(/[\\=;#\n]/g, " ")}`);
|
|
1987
|
+
t += s.durationMs;
|
|
1988
|
+
}
|
|
1989
|
+
const path = resolve(tmp, "chapters.ffmeta");
|
|
1990
|
+
await fs.writeFile(path, lines.join("\n") + "\n");
|
|
1991
|
+
log(`chapters: ${scenes.length} marker(s)`);
|
|
1992
|
+
return path;
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
async function fileToDataUri(path: string): Promise<string> {
|
|
1996
|
+
const ext = path.toLowerCase().split(".").pop() ?? "png";
|
|
1997
|
+
const mime =
|
|
1998
|
+
ext === "svg" ? "image/svg+xml" : ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
|
|
1999
|
+
const buf = await fs.readFile(path);
|
|
2000
|
+
return `data:${mime};base64,${buf.toString("base64")}`;
|
|
2001
|
+
}
|
|
2002
|
+
|
|
798
2003
|
async function muxAudio(
|
|
799
2004
|
project: Project,
|
|
800
2005
|
storyboard: Storyboard,
|
|
801
2006
|
video: string,
|
|
802
|
-
introMs: number
|
|
2007
|
+
introMs: number,
|
|
2008
|
+
chaptersPath: string | null = null
|
|
803
2009
|
): Promise<void> {
|
|
804
2010
|
let musicPath = await resolveMusic(project, storyboard);
|
|
805
2011
|
const videoMs = await probeDurationMs(video);
|
|
@@ -819,9 +2025,15 @@ async function muxAudio(
|
|
|
819
2025
|
}
|
|
820
2026
|
}
|
|
821
2027
|
|
|
822
|
-
// Inputs, fixed order: 0=video, 1=narration, 2=music (if any)
|
|
2028
|
+
// Inputs, fixed order: 0=video, 1=narration, 2=music (if any), then the
|
|
2029
|
+
// chapters metadata file (mapped by index, never a stream).
|
|
823
2030
|
const args: string[] = ["-i", video, "-i", project.narrationPath];
|
|
824
2031
|
if (musicPath) args.push("-stream_loop", String(musicLoops), "-i", musicPath);
|
|
2032
|
+
let chaptersIdx = -1;
|
|
2033
|
+
if (chaptersPath) {
|
|
2034
|
+
chaptersIdx = musicPath ? 3 : 2;
|
|
2035
|
+
args.push("-i", chaptersPath);
|
|
2036
|
+
}
|
|
825
2037
|
|
|
826
2038
|
const delay = introMs > 0 ? `,adelay=${Math.round(introMs)}:all=1` : "";
|
|
827
2039
|
const filters: string[] = [];
|
|
@@ -892,6 +2104,7 @@ async function muxAudio(
|
|
|
892
2104
|
|
|
893
2105
|
args.push("-filter_complex", filters.join(";"));
|
|
894
2106
|
args.push("-map", "0:v", "-map", "[aout]");
|
|
2107
|
+
if (chaptersIdx >= 0) args.push("-map_metadata", String(chaptersIdx));
|
|
895
2108
|
args.push(
|
|
896
2109
|
"-c:v",
|
|
897
2110
|
"copy",
|
|
@@ -899,6 +2112,10 @@ async function muxAudio(
|
|
|
899
2112
|
"aac",
|
|
900
2113
|
"-b:a",
|
|
901
2114
|
"192k",
|
|
2115
|
+
// moov atom up front so the MP4 streams/scrubs in a browser tab or a PR
|
|
2116
|
+
// preview before it has fully downloaded (frames identical; container only).
|
|
2117
|
+
"-movflags",
|
|
2118
|
+
"+faststart",
|
|
902
2119
|
"-t",
|
|
903
2120
|
durSec,
|
|
904
2121
|
project.outputPath
|