@tandryukha/aidemo 0.14.1 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/AUTHORING.md +142 -20
- package/package.json +2 -2
- package/src/captions.ts +174 -30
- package/src/cli.ts +134 -7
- package/src/compose.ts +340 -36
- package/src/cursor.ts +13 -4
- package/src/ffmpeg.ts +68 -0
- package/src/frames.ts +54 -5
- package/src/import-trace.ts +20 -9
- package/src/inspect.ts +9 -0
- package/src/lint.ts +75 -3
- package/src/mcp/jobs.ts +1 -0
- package/src/mcp/server.ts +67 -3
- package/src/player.ts +362 -45
- package/src/qa.ts +295 -0
- package/src/recorder.ts +65 -7
- package/src/redact.ts +72 -0
- package/src/safe-regex.ts +44 -0
- package/src/types.ts +155 -19
package/src/compose.ts
CHANGED
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
runFfmpeg,
|
|
51
51
|
probeDurationMs,
|
|
52
52
|
probeFlatTailMs,
|
|
53
|
+
probeFlatLeadMs,
|
|
53
54
|
probeVideoDims,
|
|
54
55
|
probeFreezeSpans,
|
|
55
56
|
type FreezeSpan,
|
|
@@ -97,6 +98,13 @@ export async function compose(
|
|
|
97
98
|
const warn = (code: string, message: string, scene?: string): void => {
|
|
98
99
|
warnings.push({ code, message, ...(scene ? { scene } : {}) });
|
|
99
100
|
};
|
|
101
|
+
/**
|
|
102
|
+
* Frames to dump next to a warning (issue #59): an agent cannot look at the
|
|
103
|
+
* MP4, so "the reward is on screen for 0.3 s" is only actionable with the
|
|
104
|
+
* frame at that moment attached. Content-time ms; the intro shift is added
|
|
105
|
+
* when they are extracted, after the final mux.
|
|
106
|
+
*/
|
|
107
|
+
const warningFrames: Array<{ name: string; atMs: number }> = [];
|
|
100
108
|
const hold = HoldSchema.parse(storyboard.hold ?? {});
|
|
101
109
|
if (await warnStaleCaptions(project)) {
|
|
102
110
|
warn(
|
|
@@ -182,6 +190,10 @@ export async function compose(
|
|
|
182
190
|
cursorCfg?.style === "dot" ? Math.round(16 * pxScale * (cursorCfg.scale ?? 1)) : 0;
|
|
183
191
|
|
|
184
192
|
const sceneVideos: string[] = [];
|
|
193
|
+
/** timeline scenes that actually produced a segment, parallel to sceneVideos. */
|
|
194
|
+
const composedTl: Timeline["scenes"] = [];
|
|
195
|
+
/** content-time span of each composed scene, for `frame.url: "auto"`. */
|
|
196
|
+
const sceneSpans: Array<{ a: number; b: number }> = [];
|
|
185
197
|
const sceneTotal = timeline.scenes.length;
|
|
186
198
|
for (let i = 0; i < timeline.scenes.length; i++) {
|
|
187
199
|
const tl = timeline.scenes[i];
|
|
@@ -217,6 +229,23 @@ export async function compose(
|
|
|
217
229
|
await extractAndConcat(sceneRaw, keeps, rawSegPath, tmp, i);
|
|
218
230
|
|
|
219
231
|
let srcMs = await probeDurationMs(rawSegPath);
|
|
232
|
+
// #47: the take opens on the browser's white pre-paint (the `goto` load),
|
|
233
|
+
// so the first caption plays over a blank page and `poster` grabs white.
|
|
234
|
+
// Trim it down to a sliver — opt-in, so an unchanged storyboard renders
|
|
235
|
+
// byte-for-byte as before.
|
|
236
|
+
let leadTrimMs = 0;
|
|
237
|
+
if (i === 0 && storyboard.output?.trimLeadingBlank && srcMs > 1000) {
|
|
238
|
+
const cap = storyboard.output.blankOpenCapMs ?? 100;
|
|
239
|
+
const flatLead = await probeFlatLeadMs(rawSegPath);
|
|
240
|
+
if (flatLead > cap + 80) {
|
|
241
|
+
leadTrimMs = Math.min(flatLead - cap, srcMs - 500);
|
|
242
|
+
srcMs -= leadTrimMs;
|
|
243
|
+
log(
|
|
244
|
+
`scene ${tl.id}: trimmed ${Math.round(leadTrimMs)}ms of blank opening ` +
|
|
245
|
+
`(kept ${cap}ms of it)`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
220
249
|
// A scene that ends right before a `goto` can trail off into the browser's
|
|
221
250
|
// white pre-paint. Freeze-holding THAT frame paints seconds of solid white
|
|
222
251
|
// (issue #36), so drop a flat tail before deciding how long to hold —
|
|
@@ -229,6 +258,13 @@ export async function compose(
|
|
|
229
258
|
srcMs = trimTo;
|
|
230
259
|
}
|
|
231
260
|
}
|
|
261
|
+
/**
|
|
262
|
+
* Raw video ms → offset inside this scene's kept, head-trimmed footage.
|
|
263
|
+
* Every event remap goes through it, so the leading-blank trim shifts the
|
|
264
|
+
* whole scene's timeline consistently.
|
|
265
|
+
*/
|
|
266
|
+
const localIn = (rawT: number): number =>
|
|
267
|
+
Math.max(0, offsetInKeeps(keeps, rawT) - leadTrimMs);
|
|
232
268
|
// Retime the segment toward the narration length, but only within
|
|
233
269
|
// [MIN_FACTOR, MAX_STRETCH]. If narration still needs more time, hold
|
|
234
270
|
// (freeze) the last frame for the remainder — natural for a static page,
|
|
@@ -252,7 +288,7 @@ export async function compose(
|
|
|
252
288
|
if (wi == null) continue;
|
|
253
289
|
const target = wordStartMs(words, sceneDef?.narration ?? "", wi);
|
|
254
290
|
if (target == null) continue;
|
|
255
|
-
const raw =
|
|
291
|
+
const raw = localIn(ev.tMs + leadInMs);
|
|
256
292
|
if (raw > srcMs - 40) {
|
|
257
293
|
warn(
|
|
258
294
|
"anchor-unreachable",
|
|
@@ -292,6 +328,10 @@ export async function compose(
|
|
|
292
328
|
offMs,
|
|
293
329
|
});
|
|
294
330
|
if (Math.abs(offMs) > 150) {
|
|
331
|
+
warningFrames.push({
|
|
332
|
+
name: `${tl.id}-${bnd.name}`,
|
|
333
|
+
atMs: outCursorMs + Math.max(0, Math.min(out, targetMs)),
|
|
334
|
+
});
|
|
295
335
|
warn(
|
|
296
336
|
"anchor-unreachable",
|
|
297
337
|
`scene ${tl.id}: anchor "${bnd.name}" lands ${Math.abs(offMs)}ms ${offMs > 0 ? "after" : "before"} its word even at ${offMs > 0 ? "2x speed" : "x1.6 slow-motion"} — ${
|
|
@@ -336,8 +376,25 @@ export async function compose(
|
|
|
336
376
|
const backoffMs = holding
|
|
337
377
|
? Math.min(hold.backoffMs, Math.max(0, stretchedMs - 500))
|
|
338
378
|
: 0;
|
|
339
|
-
|
|
379
|
+
// Drift only where it reads as a camera move (issue #57): a short hold
|
|
380
|
+
// bobs, and inside a composited window chrome the page appears to slide
|
|
381
|
+
// within a frame that does not move with it.
|
|
382
|
+
const framedChrome =
|
|
383
|
+
!!storyboard.frame && (storyboard.frame.chrome ?? "none") !== "none";
|
|
384
|
+
const driftSuppressed =
|
|
385
|
+
holding && hold.mode === "drift"
|
|
386
|
+
? holdMs < hold.minDriftMs
|
|
387
|
+
? `hold is ${Math.round(holdMs)}ms (< hold.minDriftMs ${hold.minDriftMs}ms)`
|
|
388
|
+
: framedChrome && !hold.driftInFrame
|
|
389
|
+
? `frame.chrome "${storyboard.frame!.chrome}" is static — the drift would read as the page sliding inside it (set hold.driftInFrame to override)`
|
|
390
|
+
: null
|
|
391
|
+
: null;
|
|
392
|
+
if (driftSuppressed) log(`scene ${tl.id}: drift hold → freeze (${driftSuppressed})`);
|
|
393
|
+
const driftHold = holding && hold.mode === "drift" && !driftSuppressed;
|
|
340
394
|
let vf = "";
|
|
395
|
+
if (leadTrimMs > 0) {
|
|
396
|
+
vf += `trim=start=${(leadTrimMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
|
|
397
|
+
}
|
|
341
398
|
if (blankTrimMs > 0) {
|
|
342
399
|
vf += `trim=duration=${(srcMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
|
|
343
400
|
}
|
|
@@ -382,7 +439,7 @@ export async function compose(
|
|
|
382
439
|
for (const ev of tl.focusEvents ?? []) {
|
|
383
440
|
sceneFocus++;
|
|
384
441
|
const rawT = ev.tMs + leadInMs;
|
|
385
|
-
const local = stretchLocal(
|
|
442
|
+
const local = stretchLocal(localIn(rawT));
|
|
386
443
|
zoomEvents.push({
|
|
387
444
|
tMs: outCursorMs + Math.min(local, stretchedMs),
|
|
388
445
|
x: ev.x * pxScale,
|
|
@@ -400,7 +457,7 @@ export async function compose(
|
|
|
400
457
|
const sceneStart = outCursorMs;
|
|
401
458
|
for (const s of tl.cursorSamples ?? []) {
|
|
402
459
|
const rawT = s.tMs + leadInMs;
|
|
403
|
-
const local = stretchLocal(
|
|
460
|
+
const local = stretchLocal(localIn(rawT));
|
|
404
461
|
cursorPts.push({
|
|
405
462
|
t: (outCursorMs + Math.min(local, stretchedMs)) / 1000,
|
|
406
463
|
x: s.x * pxScale - dotOffset,
|
|
@@ -420,7 +477,7 @@ export async function compose(
|
|
|
420
477
|
const sceneStart = outCursorMs;
|
|
421
478
|
const sceneEnd = outCursorMs + stretchedMs + (holdMs > 40 ? holdMs : 0);
|
|
422
479
|
const toContent = (rawMs: number): number =>
|
|
423
|
-
sceneStart + Math.min(stretchLocal(
|
|
480
|
+
sceneStart + Math.min(stretchLocal(localIn(rawMs + leadInMs)), stretchedMs);
|
|
424
481
|
for (const ev of tl.attentionEvents ?? []) {
|
|
425
482
|
const a = toContent(ev.tMs);
|
|
426
483
|
const b = Math.min(sceneEnd, a + ev.holdMs);
|
|
@@ -450,6 +507,7 @@ export async function compose(
|
|
|
450
507
|
if (pos) captionWindows.push({ a: sceneStart, b: sceneEnd, position: pos });
|
|
451
508
|
}
|
|
452
509
|
|
|
510
|
+
const sceneContentStartMs = outCursorMs;
|
|
453
511
|
outCursorMs += stretchedMs + (holdMs > 40 ? holdMs : 0);
|
|
454
512
|
|
|
455
513
|
let scenePath = resolve(tmp, `scene-${i}.mp4`);
|
|
@@ -482,12 +540,14 @@ export async function compose(
|
|
|
482
540
|
);
|
|
483
541
|
}
|
|
484
542
|
sceneVideos.push(scenePath);
|
|
543
|
+
composedTl.push(tl);
|
|
544
|
+
sceneSpans.push({ a: sceneContentStartMs, b: outCursorMs });
|
|
485
545
|
const holdPct = holding ? holdMs / targetMs : 0;
|
|
486
546
|
log(
|
|
487
547
|
`scene ${tl.id}: ${srcMs}ms -> ${targetMs}ms ` +
|
|
488
548
|
`(x${factor.toFixed(2)}${
|
|
489
549
|
holding
|
|
490
|
-
? ` + ${Math.round(holdMs)}ms ${
|
|
550
|
+
? ` + ${Math.round(holdMs)}ms ${driftHold ? "drift " : ""}hold` +
|
|
491
551
|
(backoffMs > 0 ? ` (backoff ${backoffMs}ms)` : "")
|
|
492
552
|
: ""
|
|
493
553
|
}${
|
|
@@ -514,11 +574,23 @@ export async function compose(
|
|
|
514
574
|
...(anchorReport.length ? { anchors: anchorReport } : {}),
|
|
515
575
|
});
|
|
516
576
|
if (holdPct > FREEZE_WARN_PCT) {
|
|
577
|
+
// Name the exact frame being held (issue #55): when the wrong one is
|
|
578
|
+
// frozen — the next page's load, a half-painted card — the only way to
|
|
579
|
+
// see it today is to extract frames by hand.
|
|
580
|
+
const heldRaw = rawAtKeepOffset(
|
|
581
|
+
keeps,
|
|
582
|
+
leadTrimMs + Math.max(0, srcMs - backoffMs / Math.max(factor, 0.01))
|
|
583
|
+
);
|
|
584
|
+
warningFrames.push({
|
|
585
|
+
name: `${tl.id}-held`,
|
|
586
|
+
atMs: sceneContentStartMs + stretchedMs + 200,
|
|
587
|
+
});
|
|
517
588
|
warn(
|
|
518
589
|
"scene-freeze",
|
|
519
590
|
`scene ${tl.id}: ${Math.round(holdPct * 100)}% of its ${(targetMs / 1000).toFixed(1)}s ` +
|
|
520
591
|
`is a held frame (${(srcMs / 1000).toFixed(1)}s of action for ` +
|
|
521
|
-
`${(targetMs / 1000).toFixed(1)}s of narration
|
|
592
|
+
`${(targetMs / 1000).toFixed(1)}s of narration; the held frame is ` +
|
|
593
|
+
`${relative(project.dir, sceneRaw)} @ ${stampMs(heldRaw)}) — add on-screen action ` +
|
|
522
594
|
`(hover/scroll/focus beats, a pause) or shorten the narration`,
|
|
523
595
|
tl.id
|
|
524
596
|
);
|
|
@@ -549,14 +621,43 @@ export async function compose(
|
|
|
549
621
|
// scene boundaries (re-encodes the join; total duration is preserved).
|
|
550
622
|
let content: string;
|
|
551
623
|
if (storyboard.transition && sceneVideos.length >= 2) {
|
|
552
|
-
const
|
|
624
|
+
const baseDur = storyboard.transition.durationMs;
|
|
625
|
+
// Per-boundary duration; 0 = hard cut. A boundary that crosses a page
|
|
626
|
+
// navigation dissolves the old page over a still-painting new one — a
|
|
627
|
+
// ghost double-image (issue #49) — so cut those, and honour an explicit
|
|
628
|
+
// per-scene override either way.
|
|
629
|
+
const durations: number[] = [];
|
|
630
|
+
const autoCut: string[] = [];
|
|
631
|
+
for (let b = 1; b < sceneVideos.length; b++) {
|
|
632
|
+
const scene = sceneById.get(composedTl[b].id);
|
|
633
|
+
if (scene?.transition === false) {
|
|
634
|
+
durations.push(0);
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
if (scene?.transition) {
|
|
638
|
+
durations.push(scene.transition.durationMs);
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
if (crossesNavigation(composedTl[b - 1], composedTl[b])) {
|
|
642
|
+
durations.push(0);
|
|
643
|
+
autoCut.push(composedTl[b].id);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
durations.push(baseDur);
|
|
647
|
+
}
|
|
648
|
+
const fades = durations.filter((d) => d > 0).length;
|
|
553
649
|
log(
|
|
554
|
-
`scene crossfade: ${
|
|
650
|
+
`scene crossfade: ${baseDur}ms x ${fades} boundary/ies` +
|
|
651
|
+
(durations.length - fades
|
|
652
|
+
? `, ${durations.length - fades} hard cut(s)` +
|
|
653
|
+
(autoCut.length ? ` (navigation into ${autoCut.join(", ")})` : "")
|
|
654
|
+
: "")
|
|
555
655
|
);
|
|
556
656
|
content = await crossfadeScenes(
|
|
557
657
|
sceneVideos,
|
|
558
|
-
|
|
559
|
-
resolve(tmp, "content.mp4")
|
|
658
|
+
durations,
|
|
659
|
+
resolve(tmp, "content.mp4"),
|
|
660
|
+
tmp
|
|
560
661
|
);
|
|
561
662
|
} else {
|
|
562
663
|
content = await concatSegments(
|
|
@@ -653,7 +754,12 @@ export async function compose(
|
|
|
653
754
|
png,
|
|
654
755
|
Math.round(32 * pxScale * cScale),
|
|
655
756
|
cursorCfg.style ?? "arrow",
|
|
656
|
-
cursorCfg.color
|
|
757
|
+
cursorCfg.color,
|
|
758
|
+
{
|
|
759
|
+
...(cursorCfg.outline ? { outline: cursorCfg.outline } : {}),
|
|
760
|
+
...(cursorCfg.outlineWidth != null ? { outlineWidth: cursorCfg.outlineWidth } : {}),
|
|
761
|
+
...(cursorCfg.halo ? { halo: true } : {}),
|
|
762
|
+
}
|
|
657
763
|
);
|
|
658
764
|
const withCursor = resolve(tmp, "content-cursor.mp4");
|
|
659
765
|
await runFfmpeg([
|
|
@@ -796,25 +902,49 @@ export async function compose(
|
|
|
796
902
|
let canvasH = outH;
|
|
797
903
|
const outCfg = resolveOutputSizing(storyboard.output);
|
|
798
904
|
if (storyboard.frame) {
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
905
|
+
// `frame.url: "auto"` tracks the page: one frame PNG per distinct address,
|
|
906
|
+
// time-gated to the scenes that are on it (#52). Everything else renders a
|
|
907
|
+
// single static frame, exactly as before.
|
|
908
|
+
const autoUrl = storyboard.frame.url === "auto";
|
|
909
|
+
const spans = autoUrl ? frameUrlSpans(composedTl, sceneSpans) : [];
|
|
910
|
+
const labels = autoUrl ? [...new Set(spans.map((sp) => sp.label))] : [""];
|
|
911
|
+
const framePngs: string[] = [];
|
|
912
|
+
let layout!: Awaited<ReturnType<typeof renderFramePng>>;
|
|
913
|
+
for (let li = 0; li < labels.length; li++) {
|
|
914
|
+
const framePng = resolve(tmp, `frame-${li}.png`);
|
|
915
|
+
layout = await renderFramePng(
|
|
916
|
+
autoUrl ? { ...storyboard.frame, url: labels[li] || undefined } : storyboard.frame,
|
|
917
|
+
storyboard.brand,
|
|
918
|
+
framePng,
|
|
919
|
+
storyboard.video.width,
|
|
920
|
+
storyboard.video.height,
|
|
921
|
+
pxScale,
|
|
922
|
+
outCfg.width != null && outCfg.height != null ? outCfg.width / outCfg.height : undefined
|
|
923
|
+
);
|
|
924
|
+
framePngs.push(framePng);
|
|
925
|
+
}
|
|
809
926
|
const framed = resolve(tmp, "content-framed.mp4");
|
|
927
|
+
// Each frame PNG covers the whole canvas and the spans partition the
|
|
928
|
+
// timeline, so exactly one overlay is enabled at any instant.
|
|
929
|
+
const chain = framePngs
|
|
930
|
+
.map((_, li) => {
|
|
931
|
+
const win = spans
|
|
932
|
+
.filter((sp) => sp.label === labels[li])
|
|
933
|
+
.map((sp) => `between(t,${(sp.a / 1000).toFixed(3)},${(sp.b / 1000).toFixed(3)})`)
|
|
934
|
+
.join("+");
|
|
935
|
+
const gate = autoUrl && framePngs.length > 1 ? `:enable='${win}'` : "";
|
|
936
|
+
const src = li === 0 ? "[p]" : `[v${li - 1}]`;
|
|
937
|
+
const dst = li === framePngs.length - 1 ? "[vout]" : `[v${li}]`;
|
|
938
|
+
return `${src}[${li + 1}:v]overlay=0:0:format=auto${gate}${dst}`;
|
|
939
|
+
})
|
|
940
|
+
.join(";");
|
|
810
941
|
await runFfmpeg([
|
|
811
942
|
"-i",
|
|
812
943
|
content,
|
|
813
|
-
"-i",
|
|
814
|
-
framePng,
|
|
944
|
+
...framePngs.flatMap((f) => ["-i", f]),
|
|
815
945
|
"-filter_complex",
|
|
816
946
|
`[0:v]pad=${layout.canvasW}:${layout.canvasH}:${layout.offsetX}:${layout.offsetY}:color=${layout.padColor}[p];` +
|
|
817
|
-
|
|
947
|
+
chain,
|
|
818
948
|
"-map",
|
|
819
949
|
"[vout]",
|
|
820
950
|
"-an",
|
|
@@ -835,7 +965,8 @@ export async function compose(
|
|
|
835
965
|
canvasH = layout.canvasH;
|
|
836
966
|
log(
|
|
837
967
|
`frame: ${layout.canvasW}x${layout.canvasH} canvas (padding ${storyboard.frame.padding ?? 48}` +
|
|
838
|
-
`${storyboard.frame.chrome && storyboard.frame.chrome !== "none" ? `, ${storyboard.frame.chrome} chrome` : ""})`
|
|
968
|
+
`${storyboard.frame.chrome && storyboard.frame.chrome !== "none" ? `, ${storyboard.frame.chrome} chrome` : ""})` +
|
|
969
|
+
(autoUrl ? `, auto url: ${labels.filter(Boolean).join(" -> ") || "(none)"}` : "")
|
|
839
970
|
);
|
|
840
971
|
}
|
|
841
972
|
const logicalW = canvasW / pxScale;
|
|
@@ -963,12 +1094,28 @@ export async function compose(
|
|
|
963
1094
|
: null;
|
|
964
1095
|
await muxAudio(project, storyboard, finalVideo, introMs, chapters);
|
|
965
1096
|
ok(`final video → ${project.outputPath}`);
|
|
1097
|
+
// Blank-open measurement + poster (issue #47). The poster used to be taken
|
|
1098
|
+
// at a fixed offset, which on a demo that opens with a `goto` is exactly the
|
|
1099
|
+
// white load frame; pick the first frame that actually shows content instead.
|
|
1100
|
+
const blankOpenMs = await probeFlatLeadMs(project.outputPath, 4000, introMs).catch(() => 0);
|
|
1101
|
+
if (blankOpenMs > 250) {
|
|
1102
|
+
warn(
|
|
1103
|
+
"blank-open",
|
|
1104
|
+
`the video opens on ${Math.round(blankOpenMs)}ms of blank page${storyboard.intro ? " after the intro card" : ""} ` +
|
|
1105
|
+
`(the first scene's load) — ` +
|
|
1106
|
+
`set output.trimLeadingBlank, or wait on the page's content before the first beat`
|
|
1107
|
+
);
|
|
1108
|
+
}
|
|
966
1109
|
let poster: string | undefined;
|
|
967
1110
|
if (storyboard.output?.poster) {
|
|
968
1111
|
poster = project.posterPath;
|
|
1112
|
+
const at =
|
|
1113
|
+
typeof storyboard.output.poster === "number"
|
|
1114
|
+
? storyboard.output.poster
|
|
1115
|
+
: introMs + Math.max(500, blankOpenMs + 200);
|
|
969
1116
|
await runFfmpeg([
|
|
970
1117
|
"-ss",
|
|
971
|
-
(
|
|
1118
|
+
(at / 1000).toFixed(3),
|
|
972
1119
|
"-i",
|
|
973
1120
|
project.outputPath,
|
|
974
1121
|
"-frames:v",
|
|
@@ -977,7 +1124,29 @@ export async function compose(
|
|
|
977
1124
|
"1",
|
|
978
1125
|
poster,
|
|
979
1126
|
]);
|
|
980
|
-
log(`poster → ${poster}`);
|
|
1127
|
+
log(`poster → ${poster} (@ ${stampMs(at)})`);
|
|
1128
|
+
}
|
|
1129
|
+
// Dump the queued warning frames (issue #59). Best-effort: a failed grab
|
|
1130
|
+
// must never fail a compose that otherwise produced a video.
|
|
1131
|
+
if (warningFrames.length) {
|
|
1132
|
+
const dir = project.p("output", "warnings");
|
|
1133
|
+
await ensureDir(dir);
|
|
1134
|
+
for (const f of warningFrames) {
|
|
1135
|
+
const png = resolve(dir, `${f.name}.png`);
|
|
1136
|
+
await runFfmpeg([
|
|
1137
|
+
"-y",
|
|
1138
|
+
"-ss",
|
|
1139
|
+
((introMs + f.atMs) / 1000).toFixed(3),
|
|
1140
|
+
"-i",
|
|
1141
|
+
project.outputPath,
|
|
1142
|
+
"-frames:v",
|
|
1143
|
+
"1",
|
|
1144
|
+
"-update",
|
|
1145
|
+
"1",
|
|
1146
|
+
png,
|
|
1147
|
+
]).catch(() => {});
|
|
1148
|
+
}
|
|
1149
|
+
log(`warning frames → ${relative(project.dir, dir)}/ (${warningFrames.length})`);
|
|
981
1150
|
}
|
|
982
1151
|
// AIDEMO_KEEP_TMP=1 keeps .compose-tmp for debugging intermediates.
|
|
983
1152
|
if (!process.env.AIDEMO_KEEP_TMP) {
|
|
@@ -993,6 +1162,7 @@ export async function compose(
|
|
|
993
1162
|
captions: { cues: captionsResult.cues, passes: captionsResult.passes },
|
|
994
1163
|
hold: hold.mode,
|
|
995
1164
|
...(poster ? { poster } : {}),
|
|
1165
|
+
blankOpenMs: Math.round(blankOpenMs),
|
|
996
1166
|
...(attentionPlaced.length || keysPlaced.length || clicksPlaced.length || redactPlaced.length
|
|
997
1167
|
? {
|
|
998
1168
|
attention: {
|
|
@@ -1013,8 +1183,18 @@ export async function compose(
|
|
|
1013
1183
|
(frozen ? ` (${frozen} of ${sceneReports.length} scene(s) mostly held)` : "")
|
|
1014
1184
|
);
|
|
1015
1185
|
if (warnings.length) {
|
|
1186
|
+
// Last thing on screen, grouped by scene (issue #59): in a 28-scene render
|
|
1187
|
+
// an ungrouped list scrolls away above the ✓ lines and is never read.
|
|
1016
1188
|
log(`⚠ ${warnings.length} compose warning(s):`);
|
|
1017
|
-
|
|
1189
|
+
const byScene = new Map<string, ComposeWarning[]>();
|
|
1190
|
+
for (const w of warnings) {
|
|
1191
|
+
const k = w.scene ?? "(whole video)";
|
|
1192
|
+
byScene.set(k, [...(byScene.get(k) ?? []), w]);
|
|
1193
|
+
}
|
|
1194
|
+
for (const [scene, list] of byScene) {
|
|
1195
|
+
log(` ${scene}:`);
|
|
1196
|
+
for (const w of list) log(` - [${w.code}] ${w.message}`);
|
|
1197
|
+
}
|
|
1018
1198
|
}
|
|
1019
1199
|
return report;
|
|
1020
1200
|
}
|
|
@@ -1171,12 +1351,96 @@ async function concatSegments(
|
|
|
1171
1351
|
* padded). Net effect: identical A/V duration + alignment as hard cuts, but the
|
|
1172
1352
|
* boundaries cross-dissolve. Re-encodes (xfade needs filter_complex).
|
|
1173
1353
|
*/
|
|
1354
|
+
/**
|
|
1355
|
+
* Does the boundary between these two scenes cross a page navigation? Either
|
|
1356
|
+
* the next scene opens with a load (its first idle span is the `goto` wait) or
|
|
1357
|
+
* the previous scene's last action was a navigation. Both are already in the
|
|
1358
|
+
* timeline, so no extra probing is needed.
|
|
1359
|
+
*/
|
|
1360
|
+
/**
|
|
1361
|
+
* Per-scene address-pill text for `frame.url: "auto"`: the host of the URL the
|
|
1362
|
+
* scene is on, carried forward from the last `goto` (falling back to
|
|
1363
|
+
* `baseUrl`). Consecutive scenes on the same host share one span.
|
|
1364
|
+
*/
|
|
1365
|
+
function frameUrlSpans(
|
|
1366
|
+
scenes: Timeline["scenes"],
|
|
1367
|
+
spans: Array<{ a: number; b: number }>
|
|
1368
|
+
): Array<{ a: number; b: number; label: string }> {
|
|
1369
|
+
let current = "";
|
|
1370
|
+
const out: Array<{ a: number; b: number; label: string }> = [];
|
|
1371
|
+
for (let i = 0; i < scenes.length; i++) {
|
|
1372
|
+
for (const a of scenes[i].actions ?? []) {
|
|
1373
|
+
if (a.op === "goto" && a.target) current = prettyHost(a.target) || current;
|
|
1374
|
+
}
|
|
1375
|
+
const span = spans[i];
|
|
1376
|
+
if (!span) continue;
|
|
1377
|
+
const last = out.at(-1);
|
|
1378
|
+
if (last && last.label === current) last.b = span.b;
|
|
1379
|
+
else out.push({ a: i === 0 ? 0 : span.a, b: span.b, label: current });
|
|
1380
|
+
}
|
|
1381
|
+
if (out.length) out[out.length - 1].b += 3600_000; // cover holds/cards past the last scene
|
|
1382
|
+
return out;
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
function prettyHost(url?: string): string {
|
|
1386
|
+
if (!url) return "";
|
|
1387
|
+
try {
|
|
1388
|
+
const u = new URL(url, "http://localhost");
|
|
1389
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") return "";
|
|
1390
|
+
return u.host.replace(/^www\./, "") + (u.pathname !== "/" ? u.pathname : "");
|
|
1391
|
+
} catch {
|
|
1392
|
+
return "";
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
function crossesNavigation(
|
|
1397
|
+
prev: Timeline["scenes"][number],
|
|
1398
|
+
next: Timeline["scenes"][number]
|
|
1399
|
+
): boolean {
|
|
1400
|
+
const opensOnLoad = (next.idleSpans ?? []).some(
|
|
1401
|
+
(sp) => sp.label === "load" && sp.startMs - next.startMs < 1500
|
|
1402
|
+
);
|
|
1403
|
+
if (opensOnLoad) return true;
|
|
1404
|
+
const lastOp = (prev.actions ?? []).filter((a) => !a.skipped).at(-1)?.op;
|
|
1405
|
+
const firstOp = (next.actions ?? [])[0]?.op;
|
|
1406
|
+
return lastOp === "goto" || lastOp === "back" || firstOp === "goto" || firstOp === "back";
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1174
1409
|
async function crossfadeScenes(
|
|
1175
1410
|
sceneVideos: string[],
|
|
1176
|
-
|
|
1177
|
-
outPath: string
|
|
1411
|
+
durationsMs: number[],
|
|
1412
|
+
outPath: string,
|
|
1413
|
+
tmp: string
|
|
1178
1414
|
): Promise<string> {
|
|
1179
|
-
|
|
1415
|
+
// Hard-cut boundaries split the run into groups: each group is crossfaded on
|
|
1416
|
+
// its own (duration is preserved inside a group) and the groups are then
|
|
1417
|
+
// concatenated, which IS the hard cut.
|
|
1418
|
+
if (durationsMs.some((d) => d <= 0)) {
|
|
1419
|
+
const groups: string[][] = [[sceneVideos[0]]];
|
|
1420
|
+
const groupDurs: number[][] = [[]];
|
|
1421
|
+
for (let i = 1; i < sceneVideos.length; i++) {
|
|
1422
|
+
if (durationsMs[i - 1] <= 0) {
|
|
1423
|
+
groups.push([sceneVideos[i]]);
|
|
1424
|
+
groupDurs.push([]);
|
|
1425
|
+
} else {
|
|
1426
|
+
groups[groups.length - 1].push(sceneVideos[i]);
|
|
1427
|
+
groupDurs[groupDurs.length - 1].push(durationsMs[i - 1]);
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
const parts: string[] = [];
|
|
1431
|
+
for (let g = 0; g < groups.length; g++) {
|
|
1432
|
+
if (groups[g].length === 1) {
|
|
1433
|
+
parts.push(groups[g][0]);
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
parts.push(
|
|
1437
|
+
await crossfadeScenes(groups[g], groupDurs[g], resolve(tmp, `xfade-${g}.mp4`), tmp)
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
if (parts.length === 1) return parts[0];
|
|
1441
|
+
return concatSegments(parts, outPath, tmp, "xfade-groups", true);
|
|
1442
|
+
}
|
|
1443
|
+
const D = Math.max(0.05, Math.max(...durationsMs) / 1000);
|
|
1180
1444
|
// Measure each segment so the offsets track real (not assumed) durations.
|
|
1181
1445
|
const durs = await Promise.all(sceneVideos.map((p) => probeDurationMs(p)));
|
|
1182
1446
|
// Freeze-tail margin: a touch longer than D so the fade never starves at the
|
|
@@ -1205,8 +1469,9 @@ async function crossfadeScenes(
|
|
|
1205
1469
|
cumMs += durs[i - 1];
|
|
1206
1470
|
const offset = (cumMs / 1000).toFixed(3);
|
|
1207
1471
|
const out = i === sceneVideos.length - 1 ? "xout" : `x${i}`;
|
|
1472
|
+
const d = Math.max(0.05, durationsMs[i - 1] / 1000);
|
|
1208
1473
|
filters.push(
|
|
1209
|
-
`[${prev}][${labels[i]}]xfade=transition=fade:duration=${
|
|
1474
|
+
`[${prev}][${labels[i]}]xfade=transition=fade:duration=${d.toFixed(
|
|
1210
1475
|
3
|
|
1211
1476
|
)}:offset=${offset}[${out}]`
|
|
1212
1477
|
);
|
|
@@ -1362,11 +1627,29 @@ function sceneAutoIdle(
|
|
|
1362
1627
|
const EDGE_MS = 300;
|
|
1363
1628
|
const lo = tl.startMs + leadInMs + EDGE_MS;
|
|
1364
1629
|
const hi = tl.endMs + leadInMs - EDGE_MS;
|
|
1630
|
+
// Windows the author explicitly asked to be ON SCREEN: an attention overlay
|
|
1631
|
+
// and its hold, a deliberate `focus`, a named still. A static confirmation
|
|
1632
|
+
// card is motionless by nature, so freezedetect happily proposes trimming
|
|
1633
|
+
// exactly the beat the scene exists for — and compose then holds the NEXT
|
|
1634
|
+
// page's frame under that scene's narration (issue #55).
|
|
1635
|
+
const protectedWindows: Array<[number, number]> = [
|
|
1636
|
+
...(tl.attentionEvents ?? []).map(
|
|
1637
|
+
(e): [number, number] => [e.tMs + leadInMs, e.tMs + e.holdMs + leadInMs]
|
|
1638
|
+
),
|
|
1639
|
+
...(tl.focusEvents ?? [])
|
|
1640
|
+
.filter((e) => e.kind === "focus")
|
|
1641
|
+
.map((e): [number, number] => [e.tMs + leadInMs, e.tMs + (e.holdMs ?? 1200) + leadInMs]),
|
|
1642
|
+
...(tl.stillEvents ?? []).map(
|
|
1643
|
+
(e): [number, number] => [e.tMs + leadInMs - 200, e.tMs + leadInMs + 200]
|
|
1644
|
+
),
|
|
1645
|
+
];
|
|
1365
1646
|
const out: IdleSpan[] = [];
|
|
1366
1647
|
for (const f of freezes) {
|
|
1367
1648
|
const a = Math.max(f.startMs, lo);
|
|
1368
1649
|
const b = Math.min(f.endMs, hi);
|
|
1369
|
-
if (b - a
|
|
1650
|
+
if (b - a <= 200) continue;
|
|
1651
|
+
if (protectedWindows.some(([pa, pb]) => pa < b && pb > a)) continue;
|
|
1652
|
+
out.push({ startMs: a - leadInMs, endMs: b - leadInMs, label: "auto-idle" });
|
|
1370
1653
|
}
|
|
1371
1654
|
return out;
|
|
1372
1655
|
}
|
|
@@ -1425,6 +1708,24 @@ function offsetInKeeps(keeps: Array<[number, number]>, t: number): number {
|
|
|
1425
1708
|
return off;
|
|
1426
1709
|
}
|
|
1427
1710
|
|
|
1711
|
+
/** Inverse of `offsetInKeeps`: kept-span offset → raw video ms. */
|
|
1712
|
+
function rawAtKeepOffset(keeps: Array<[number, number]>, local: number): number {
|
|
1713
|
+
let off = 0;
|
|
1714
|
+
for (const [a, b] of keeps) {
|
|
1715
|
+
const len = b - a;
|
|
1716
|
+
if (local <= off + len) return a + (local - off);
|
|
1717
|
+
off += len;
|
|
1718
|
+
}
|
|
1719
|
+
return keeps.length ? keeps[keeps.length - 1][1] : 0;
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
/** mm:ss.t for a ms offset, for log lines that point back into the raw take. */
|
|
1723
|
+
function stampMs(ms: number): string {
|
|
1724
|
+
const s = Math.max(0, ms) / 1000;
|
|
1725
|
+
const m = Math.floor(s / 60);
|
|
1726
|
+
return `${m}:${(s - m * 60).toFixed(1).padStart(4, "0")}`;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1428
1729
|
/** Extract each keep span from raw and concat into one mp4 (uniform h264). */
|
|
1429
1730
|
async function extractAndConcat(
|
|
1430
1731
|
rawVideo: string,
|
|
@@ -1659,13 +1960,16 @@ async function burnCaptions(
|
|
|
1659
1960
|
*/
|
|
1660
1961
|
/** Output sizing with a preset's defaults filled in (explicit keys win). */
|
|
1661
1962
|
function resolveOutputSizing(output: Output | undefined): Output {
|
|
1662
|
-
if (!output) return {
|
|
1963
|
+
if (!output) return {};
|
|
1663
1964
|
const p = output.preset ? OUTPUT_PRESETS[output.preset] : null;
|
|
1664
1965
|
return {
|
|
1665
1966
|
...output,
|
|
1666
1967
|
width: output.width ?? p?.width,
|
|
1667
1968
|
height: output.height ?? p?.height,
|
|
1668
|
-
fit
|
|
1969
|
+
// An explicit fit wins over the preset's (issue #58): `preset: "youtube"`
|
|
1970
|
+
// + `fit: "cover"` is how you get an edge-to-edge 16:9 cut from a 16:10
|
|
1971
|
+
// capture, and before this the preset silently forced "contain".
|
|
1972
|
+
fit: output.fit ?? p?.fit ?? "contain",
|
|
1669
1973
|
};
|
|
1670
1974
|
}
|
|
1671
1975
|
|
package/src/cursor.ts
CHANGED
|
@@ -24,7 +24,8 @@ export async function renderCursorPng(
|
|
|
24
24
|
outPath: string,
|
|
25
25
|
sizePx: number,
|
|
26
26
|
style: "arrow" | "dot" = "arrow",
|
|
27
|
-
color = "rgba(255,90,95,.85)"
|
|
27
|
+
color = "rgba(255,90,95,.85)",
|
|
28
|
+
look: { outline?: string; outlineWidth?: number; halo?: boolean } = {}
|
|
28
29
|
): Promise<void> {
|
|
29
30
|
// Logical canvas is 32 units (24 arrow + shadow headroom); scale to sizePx.
|
|
30
31
|
const LOGICAL = 32;
|
|
@@ -40,15 +41,23 @@ export async function renderCursorPng(
|
|
|
40
41
|
`<!doctype html><html><head><meta charset="utf-8"><style>
|
|
41
42
|
html,body{margin:0;padding:0;background:transparent;}
|
|
42
43
|
svg{position:absolute;left:0;top:0;
|
|
43
|
-
filter
|
|
44
|
+
filter:${
|
|
45
|
+
look.halo
|
|
46
|
+
? "drop-shadow(0 0 3px rgba(0,0,0,.65)) drop-shadow(0 0 7px rgba(0,0,0,.45))"
|
|
47
|
+
: "drop-shadow(0 1px 2px rgba(0,0,0,.45))"
|
|
48
|
+
};}
|
|
44
49
|
</style></head><body>
|
|
45
50
|
<svg width="${LOGICAL}" height="${LOGICAL}" viewBox="0 0 ${LOGICAL} ${LOGICAL}"
|
|
46
51
|
xmlns="http://www.w3.org/2000/svg">
|
|
47
52
|
${
|
|
48
53
|
style === "dot"
|
|
49
|
-
? `<circle cx="16" cy="16" r="9" fill="${color}" stroke="
|
|
54
|
+
? `<circle cx="16" cy="16" r="9" fill="${color}" stroke="${
|
|
55
|
+
look.outline ?? "rgba(255,255,255,.9)"
|
|
56
|
+
}" stroke-width="${look.outlineWidth ?? 2}"/>`
|
|
50
57
|
: `<path d="${CURSOR_ARROW_PATH}" fill="${CURSOR_FILL}"
|
|
51
|
-
stroke="${CURSOR_STROKE}" stroke-width="
|
|
58
|
+
stroke="${look.outline ?? CURSOR_STROKE}" stroke-width="${
|
|
59
|
+
look.outlineWidth ?? 1.3
|
|
60
|
+
}" stroke-linejoin="round"/>`
|
|
52
61
|
}
|
|
53
62
|
</svg>
|
|
54
63
|
</body></html>`,
|