@hyperframes/engine 0.6.0-alpha.2 → 0.6.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +22 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +3 -1
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/services/audioMixer.d.ts.map +1 -1
- package/dist/services/audioMixer.js +4 -6
- package/dist/services/audioMixer.js.map +1 -1
- package/dist/services/browserManager.d.ts +35 -0
- package/dist/services/browserManager.d.ts.map +1 -1
- package/dist/services/browserManager.js +113 -1
- package/dist/services/browserManager.js.map +1 -1
- package/dist/services/chunkEncoder.d.ts.map +1 -1
- package/dist/services/chunkEncoder.js +31 -0
- package/dist/services/chunkEncoder.js.map +1 -1
- package/dist/services/frameCapture.d.ts.map +1 -1
- package/dist/services/frameCapture.js +26 -12
- package/dist/services/frameCapture.js.map +1 -1
- package/dist/services/screenshotService.d.ts.map +1 -1
- package/dist/services/screenshotService.js +7 -0
- package/dist/services/screenshotService.js.map +1 -1
- package/dist/services/streamingEncoder.d.ts.map +1 -1
- package/dist/services/streamingEncoder.js +22 -0
- package/dist/services/streamingEncoder.js.map +1 -1
- package/dist/services/videoFrameExtractor.d.ts +20 -0
- package/dist/services/videoFrameExtractor.d.ts.map +1 -1
- package/dist/services/videoFrameExtractor.js +95 -7
- package/dist/services/videoFrameExtractor.js.map +1 -1
- package/dist/services/videoFrameInjector.d.ts +40 -1
- package/dist/services/videoFrameInjector.d.ts.map +1 -1
- package/dist/services/videoFrameInjector.js +64 -9
- package/dist/services/videoFrameInjector.js.map +1 -1
- package/dist/utils/alphaBlit.d.ts +1 -1
- package/dist/utils/alphaBlit.d.ts.map +1 -1
- package/dist/utils/alphaBlit.js +15 -6
- package/dist/utils/alphaBlit.js.map +1 -1
- package/dist/utils/ffprobe.d.ts.map +1 -1
- package/dist/utils/ffprobe.js +17 -1
- package/dist/utils/ffprobe.js.map +1 -1
- package/package.json +2 -2
- package/src/config.test.ts +7 -0
- package/src/config.ts +31 -4
- package/src/index.ts +2 -0
- package/src/services/audioMixer.ts +4 -6
- package/src/services/browserManager.test.ts +83 -2
- package/src/services/browserManager.ts +130 -1
- package/src/services/chunkEncoder.ts +36 -0
- package/src/services/frameCapture.ts +26 -11
- package/src/services/screenshotService.test.ts +92 -0
- package/src/services/screenshotService.ts +8 -0
- package/src/services/streamingEncoder.ts +28 -0
- package/src/services/videoFrameExtractor.test.ts +117 -1
- package/src/services/videoFrameExtractor.ts +100 -7
- package/src/services/videoFrameInjector.test.ts +145 -0
- package/src/services/videoFrameInjector.ts +89 -11
- package/src/utils/alphaBlit.test.ts +10 -0
- package/src/utils/alphaBlit.ts +15 -7
- package/src/utils/ffprobe.test.ts +40 -0
- package/src/utils/ffprobe.ts +16 -1
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { spawn } from "child_process";
|
|
9
9
|
import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
|
|
10
|
-
import { isAbsolute, join } from "path";
|
|
10
|
+
import { isAbsolute, join, posix, resolve, sep } from "path";
|
|
11
11
|
import { parseHTML } from "linkedom";
|
|
12
12
|
import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
|
|
13
13
|
import {
|
|
@@ -230,8 +230,17 @@ export async function extractVideoFramesRange(
|
|
|
230
230
|
if (isHdr && isMacOS) {
|
|
231
231
|
args.push("-hwaccel", "videotoolbox");
|
|
232
232
|
}
|
|
233
|
-
|
|
234
|
-
|
|
233
|
+
// Always force the alpha-aware decoder on codecs that can carry alpha. The
|
|
234
|
+
// alternative — gating on `metadata.hasAlpha` — relies on tag detection that
|
|
235
|
+
// has at least three known failure modes: case-sensitivity across ffmpeg
|
|
236
|
+
// versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
|
|
237
|
+
// and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
|
|
238
|
+
// silently strips alpha during decode and the bug doesn't surface until
|
|
239
|
+
// the rendered video is missing layers. Codec-based default has no such
|
|
240
|
+
// ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
|
|
241
|
+
// normally when it isn't.
|
|
242
|
+
if (codecMayHaveAlpha(metadata.videoCodec)) {
|
|
243
|
+
args.push("-c:v", decoderForCodec(metadata.videoCodec));
|
|
235
244
|
}
|
|
236
245
|
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
|
237
246
|
|
|
@@ -398,9 +407,31 @@ function resolveSegmentDuration(
|
|
|
398
407
|
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
|
|
399
408
|
}
|
|
400
409
|
|
|
410
|
+
/**
|
|
411
|
+
* Codecs whose bitstream is allowed to carry an alpha channel. Default the
|
|
412
|
+
* extraction path to PNG output for these regardless of `metadata.hasAlpha`
|
|
413
|
+
* so a missed sidecar tag doesn't silently strip transparency. Opaque content
|
|
414
|
+
* encoded in one of these codecs pays a small file-size cost on the cached
|
|
415
|
+
* frames but stays correct on the rare case where alpha IS present and the
|
|
416
|
+
* tag was missed.
|
|
417
|
+
*/
|
|
418
|
+
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
|
|
419
|
+
|
|
420
|
+
export function codecMayHaveAlpha(codec: string | undefined): boolean {
|
|
421
|
+
return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
export function decoderForCodec(codec: string | undefined): string {
|
|
425
|
+
const c = (codec ?? "").toLowerCase();
|
|
426
|
+
if (c === "vp9") return "libvpx-vp9";
|
|
427
|
+
if (c === "vp8") return "libvpx";
|
|
428
|
+
return c;
|
|
429
|
+
}
|
|
430
|
+
|
|
401
431
|
function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
|
|
402
432
|
if (requested) return requested;
|
|
403
|
-
|
|
433
|
+
if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
|
|
434
|
+
return "jpg";
|
|
404
435
|
}
|
|
405
436
|
|
|
406
437
|
/**
|
|
@@ -459,6 +490,54 @@ async function convertVfrToCfr(
|
|
|
459
490
|
}
|
|
460
491
|
}
|
|
461
492
|
|
|
493
|
+
/**
|
|
494
|
+
* Resolve a relative `<video src>` to a filesystem path the way the browser
|
|
495
|
+
* resolves it as a URL. Browsers clamp `..` segments at the served origin's
|
|
496
|
+
* root; `path.join(projectDir, "../assets/foo")` does not. So a sub-comp
|
|
497
|
+
* `<video src="../assets/foo">` loads in the page (browser clamps to
|
|
498
|
+
* `<projectDir>/assets/foo`) but the filesystem-side resolver lands at
|
|
499
|
+
* `<parentOfProjectDir>/assets/foo` — file missing, extraction skipped,
|
|
500
|
+
* the rendered output shows the video's first frame for the whole clip.
|
|
501
|
+
*
|
|
502
|
+
* The clamp covers two escape patterns: leading `..` (`../assets/foo`) AND
|
|
503
|
+
* mid-path escapes (`assets/../../foo`) that `path.join` collapses past the
|
|
504
|
+
* project root silently. Both fall back to a project-rooted candidate that
|
|
505
|
+
* strips traversal from the resolved path.
|
|
506
|
+
*
|
|
507
|
+
* Returns the first existing candidate, or the base-dir join on miss so
|
|
508
|
+
* the caller's `existsSync` check produces a stable error path.
|
|
509
|
+
*/
|
|
510
|
+
export function resolveProjectRelativeSrc(
|
|
511
|
+
src: string,
|
|
512
|
+
baseDir: string,
|
|
513
|
+
compiledDir?: string,
|
|
514
|
+
): string {
|
|
515
|
+
const fromCompiled = compiledDir ? join(compiledDir, src) : null;
|
|
516
|
+
const fromBase = join(baseDir, src);
|
|
517
|
+
const candidates: string[] = [];
|
|
518
|
+
if (fromCompiled) candidates.push(fromCompiled);
|
|
519
|
+
candidates.push(fromBase);
|
|
520
|
+
// If the joined result escapes the project root (either via leading `..`
|
|
521
|
+
// or mid-path traversal that path.join collapsed past baseDir), retry
|
|
522
|
+
// with the basename re-anchored at the project root. This mirrors the
|
|
523
|
+
// browser URL clamp without relying on a particular `..` shape.
|
|
524
|
+
const baseAbs = resolve(baseDir);
|
|
525
|
+
const fromBaseAbs = resolve(fromBase);
|
|
526
|
+
if (!fromBaseAbs.startsWith(baseAbs + sep) && fromBaseAbs !== baseAbs) {
|
|
527
|
+
// Normalize first (`assets/../../assets/foo.mp4` → `../assets/foo.mp4`)
|
|
528
|
+
// then strip any remaining leading `..` segments. Stripping `..` from the
|
|
529
|
+
// raw input would leave dangling siblings (`assets/../../assets/foo`
|
|
530
|
+
// would become `assets/assets/foo` instead of `assets/foo`).
|
|
531
|
+
const normalized = posix.normalize(src.replace(/\\/g, "/"));
|
|
532
|
+
const stripped = normalized.replace(/^(\.\.\/)+/, "");
|
|
533
|
+
if (stripped && stripped !== src && !stripped.startsWith("..")) {
|
|
534
|
+
if (compiledDir) candidates.push(join(compiledDir, stripped));
|
|
535
|
+
candidates.push(join(baseDir, stripped));
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return candidates.find(existsSync) ?? fromBase;
|
|
539
|
+
}
|
|
540
|
+
|
|
462
541
|
export async function extractAllVideoFrames(
|
|
463
542
|
videos: VideoElement[],
|
|
464
543
|
baseDir: string,
|
|
@@ -487,6 +566,9 @@ export async function extractAllVideoFrames(
|
|
|
487
566
|
// Phase 1: Resolve paths and download remote videos
|
|
488
567
|
const phase1Start = Date.now();
|
|
489
568
|
const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
|
|
569
|
+
// Dedupe missing-src warnings: a composition with N <video> elements all
|
|
570
|
+
// pointing at the same broken src should only print one warning, not N.
|
|
571
|
+
const warnedSrcs = new Set<string>();
|
|
490
572
|
for (const video of videos) {
|
|
491
573
|
if (signal?.aborted) break;
|
|
492
574
|
try {
|
|
@@ -496,9 +578,7 @@ export async function extractAllVideoFrames(
|
|
|
496
578
|
// baseDir and produce duplicated, nonexistent paths
|
|
497
579
|
// (e.g. C:\tmp\hf-vfr-test-X\C:\tmp\hf-vfr-test-X\vfr_screen.mp4).
|
|
498
580
|
if (!isAbsolute(videoPath) && !isHttpUrl(videoPath)) {
|
|
499
|
-
|
|
500
|
-
videoPath =
|
|
501
|
-
fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
|
|
581
|
+
videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
|
|
502
582
|
}
|
|
503
583
|
|
|
504
584
|
if (isHttpUrl(videoPath)) {
|
|
@@ -508,6 +588,19 @@ export async function extractAllVideoFrames(
|
|
|
508
588
|
}
|
|
509
589
|
|
|
510
590
|
if (!existsSync(videoPath)) {
|
|
591
|
+
// Loud: silent miss leaves the rendered video frozen at frame 0 with
|
|
592
|
+
// no error in stdout — extremely confusing for authors. Dedupe by
|
|
593
|
+
// src so 50 broken videos pointing at the same path don't spam.
|
|
594
|
+
if (!warnedSrcs.has(video.src)) {
|
|
595
|
+
warnedSrcs.add(video.src);
|
|
596
|
+
process.stderr.write(
|
|
597
|
+
`[hyperframes:render] WARNING: video src="${video.src}" ` +
|
|
598
|
+
`could not be resolved on disk (looked for ${videoPath}). ` +
|
|
599
|
+
`The rendered output will show this video's first frame for the entire clip duration. ` +
|
|
600
|
+
`If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
|
|
601
|
+
`(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
|
|
602
|
+
);
|
|
603
|
+
}
|
|
511
604
|
errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
|
|
512
605
|
continue;
|
|
513
606
|
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// @vitest-environment node
|
|
2
|
+
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
3
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { __testing } from "./videoFrameInjector.js";
|
|
7
|
+
import { DEFAULT_CONFIG } from "../config.js";
|
|
8
|
+
|
|
9
|
+
const { createFrameSourceCache } = __testing;
|
|
10
|
+
|
|
11
|
+
const SHARED_STATS = { evictions: 0, oversizedRejections: 0 };
|
|
12
|
+
|
|
13
|
+
describe("frame source cache eviction", () => {
|
|
14
|
+
let dir: string;
|
|
15
|
+
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
dir = mkdtempSync(join(tmpdir(), "hf-frame-cache-test-"));
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
afterEach(() => {
|
|
21
|
+
rmSync(dir, { recursive: true, force: true });
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
// Each PNG is base64-encoded into the data URI, so the cached string is
|
|
25
|
+
// ~4/3 the file size plus a small `data:image/png;base64,` prefix. Build
|
|
26
|
+
// distinct files so eviction has predictable victims.
|
|
27
|
+
function writeFrame(name: string, sizeBytes: number): string {
|
|
28
|
+
const filePath = join(dir, name);
|
|
29
|
+
writeFileSync(filePath, Buffer.alloc(sizeBytes, 0));
|
|
30
|
+
return filePath;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
it("evicts oldest entry when entry count exceeds limit", async () => {
|
|
34
|
+
const cache = createFrameSourceCache(2, Number.MAX_SAFE_INTEGER);
|
|
35
|
+
const a = writeFrame("a.png", 16);
|
|
36
|
+
const b = writeFrame("b.png", 16);
|
|
37
|
+
const c = writeFrame("c.png", 16);
|
|
38
|
+
|
|
39
|
+
await cache.get(a);
|
|
40
|
+
await cache.get(b);
|
|
41
|
+
expect(cache.stats().entries).toBe(2);
|
|
42
|
+
|
|
43
|
+
await cache.get(c);
|
|
44
|
+
expect(cache.stats().entries).toBe(2);
|
|
45
|
+
expect(cache.stats().evictions).toBe(1);
|
|
46
|
+
|
|
47
|
+
// Verify the *oldest* entry (a) was the victim — the LRU contract.
|
|
48
|
+
// A later get(a) is a miss-then-insert, which would also evict whichever
|
|
49
|
+
// entry is now oldest. We instrument the eviction counter to detect it.
|
|
50
|
+
const evictionsBefore = cache.stats().evictions;
|
|
51
|
+
await cache.get(a);
|
|
52
|
+
expect(cache.stats().evictions).toBe(evictionsBefore + 1);
|
|
53
|
+
// After re-inserting `a`, `b` is the next oldest. `c` is now newest.
|
|
54
|
+
// Touch `b` (move-to-front) → next eviction would be `c`, not `b`.
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("evicts oldest entry when byte budget is exceeded", async () => {
|
|
58
|
+
// 1 KB raw frame → ~1.4 KB base64 + ~22-byte data URI prefix. Pick a
|
|
59
|
+
// budget that comfortably fits two URIs but not three, so the third
|
|
60
|
+
// get() forces eviction even though the entry-count cap (100) is far
|
|
61
|
+
// from the limit.
|
|
62
|
+
const cache = createFrameSourceCache(100, 4 * 1024);
|
|
63
|
+
const a = writeFrame("a.png", 1024);
|
|
64
|
+
const b = writeFrame("b.png", 1024);
|
|
65
|
+
const c = writeFrame("c.png", 1024);
|
|
66
|
+
|
|
67
|
+
await cache.get(a);
|
|
68
|
+
await cache.get(b);
|
|
69
|
+
expect(cache.stats().entries).toBe(2);
|
|
70
|
+
|
|
71
|
+
await cache.get(c);
|
|
72
|
+
const afterC = cache.stats();
|
|
73
|
+
// The byte budget is the contract — the cache MUST stay under it after
|
|
74
|
+
// an insert that would otherwise overflow. Entry count is incidental.
|
|
75
|
+
expect(afterC.bytes).toBeLessThanOrEqual(4 * 1024);
|
|
76
|
+
expect(afterC.entries).toBeLessThan(3);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it("returns the served URL untouched when frameSrcResolver yields one", async () => {
|
|
80
|
+
let served: string | null = "/served/frame.png";
|
|
81
|
+
const cache = createFrameSourceCache(4, 64 * 1024, () => served);
|
|
82
|
+
const file = writeFrame("a.png", 256);
|
|
83
|
+
|
|
84
|
+
expect(await cache.get(file)).toBe("/served/frame.png");
|
|
85
|
+
// Cache stays empty because the resolver short-circuits the read.
|
|
86
|
+
expect(cache.stats()).toMatchObject({ entries: 0, bytes: 0 });
|
|
87
|
+
|
|
88
|
+
served = null;
|
|
89
|
+
const dataUri = await cache.get(file);
|
|
90
|
+
expect(dataUri.startsWith("data:image/png;base64,")).toBe(true);
|
|
91
|
+
expect(cache.stats().entries).toBe(1);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("treats a re-read as a cache hit (no second file read)", async () => {
|
|
95
|
+
const cache = createFrameSourceCache(2, Number.MAX_SAFE_INTEGER);
|
|
96
|
+
const a = writeFrame("a.png", 64);
|
|
97
|
+
|
|
98
|
+
const first = await cache.get(a);
|
|
99
|
+
const second = await cache.get(a);
|
|
100
|
+
expect(second).toBe(first);
|
|
101
|
+
expect(cache.stats().entries).toBe(1);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("skips caching an entry that alone exceeds the byte budget (no self-eviction)", async () => {
|
|
105
|
+
// 64 KB raw → ~88 KB base64 + prefix. Budget of 32 KB rejects this entry.
|
|
106
|
+
// The contract: caller still gets the data URI; cache stays empty so
|
|
107
|
+
// future inserts aren't blocked by the rejected entry's bookkeeping.
|
|
108
|
+
const cache = createFrameSourceCache(100, 32 * 1024);
|
|
109
|
+
const big = writeFrame("big.png", 64 * 1024);
|
|
110
|
+
|
|
111
|
+
const dataUri = await cache.get(big);
|
|
112
|
+
expect(dataUri.startsWith("data:image/png;base64,")).toBe(true);
|
|
113
|
+
expect(cache.stats().entries).toBe(0);
|
|
114
|
+
expect(cache.stats().bytes).toBe(0);
|
|
115
|
+
expect(cache.stats().oversizedRejections).toBe(1);
|
|
116
|
+
expect(cache.stats().evictions).toBe(0);
|
|
117
|
+
|
|
118
|
+
// A subsequent normal-sized entry must cache cleanly — the rejection
|
|
119
|
+
// path didn't pollute internal state.
|
|
120
|
+
const small = writeFrame("small.png", 1024);
|
|
121
|
+
await cache.get(small);
|
|
122
|
+
expect(cache.stats().entries).toBe(1);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("at the production default (1500 MB), 1080p frames stay cached", async () => {
|
|
126
|
+
// Regression for the post-PR-#662 default: previously the cache held up
|
|
127
|
+
// to 256 entries × ~8 MB ≈ 2 GB at 1080p. The new byte-budget default of
|
|
128
|
+
// 1500 MB caps it tighter (~187 entries at 1080p ≈ 6s @ 30fps). This
|
|
129
|
+
// test pins the math so a future tweak to the default is visible.
|
|
130
|
+
const oneEightyP_jpegSize = 8 * 1024 * 1024; // ~8 MB JPEG (data URI)
|
|
131
|
+
const defaultBytesLimit = DEFAULT_CONFIG.frameDataUriCacheBytesLimitMb * 1024 * 1024;
|
|
132
|
+
const expectedMaxEntries = Math.floor(defaultBytesLimit / oneEightyP_jpegSize);
|
|
133
|
+
expect(expectedMaxEntries).toBeGreaterThanOrEqual(180);
|
|
134
|
+
expect(expectedMaxEntries).toBeLessThanOrEqual(200);
|
|
135
|
+
// At 30fps that's at least 6 seconds of look-ahead. Sequential access is
|
|
136
|
+
// strictly cheaper, so the cache helps any seek-back ≤ 6s.
|
|
137
|
+
expect(expectedMaxEntries / 30).toBeGreaterThanOrEqual(6);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
// Suppress unused-import warning when the SHARED_STATS sentinel is dropped.
|
|
141
|
+
it("stats() exposes counters used by telemetry", async () => {
|
|
142
|
+
const cache = createFrameSourceCache(1, Number.MAX_SAFE_INTEGER);
|
|
143
|
+
expect(cache.stats()).toMatchObject({ ...SHARED_STATS, entries: 0, bytes: 0 });
|
|
144
|
+
});
|
|
145
|
+
});
|
|
@@ -15,28 +15,91 @@ import { type BeforeCaptureHook } from "./frameCapture.js";
|
|
|
15
15
|
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
|
16
16
|
|
|
17
17
|
export interface VideoFrameInjectorOptions extends Partial<
|
|
18
|
-
Pick<EngineConfig, "frameDataUriCacheLimit">
|
|
18
|
+
Pick<EngineConfig, "frameDataUriCacheLimit" | "frameDataUriCacheBytesLimitMb">
|
|
19
19
|
> {
|
|
20
20
|
frameSrcResolver?: (framePath: string) => string | null;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
interface FrameSourceCacheStats {
|
|
24
|
+
entries: number;
|
|
25
|
+
bytes: number;
|
|
26
|
+
/** Total entries evicted since cache creation. A high count vs a small
|
|
27
|
+
* composition signals the byte budget is too tight (cache thrash). */
|
|
28
|
+
evictions: number;
|
|
29
|
+
/** Total inserts rejected because the entry alone exceeds bytesLimit.
|
|
30
|
+
* Non-zero means a single frame is bigger than the configured budget —
|
|
31
|
+
* raise `frameDataUriCacheBytesLimitMb` if it recurs in production. */
|
|
32
|
+
oversizedRejections: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface FrameSourceCache {
|
|
36
|
+
get: (framePath: string) => Promise<string>;
|
|
37
|
+
/** Exposed for tests + telemetry; reflects current cache occupancy. */
|
|
38
|
+
stats: () => FrameSourceCacheStats;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Two-bound LRU keyed by frame path. Either bound triggers eviction of the
|
|
43
|
+
* oldest entry — entry count protects against pathological many-tiny-frames
|
|
44
|
+
* cases, and the byte budget keeps memory bounded when the per-frame data
|
|
45
|
+
* URI grows (4K PNG frames are ~33 MB once base64-encoded).
|
|
46
|
+
*
|
|
47
|
+
* If a single entry's data URI exceeds `bytesLimit`, we skip caching it
|
|
48
|
+
* (returning the URI directly to the caller). Without this guard, the
|
|
49
|
+
* post-insert eviction loop would drop the entry we just inserted and the
|
|
50
|
+
* cache would degrade into a CPU hot path — every subsequent `get()` would
|
|
51
|
+
* re-read from disk and re-base64 the same frame.
|
|
52
|
+
*
|
|
53
|
+
* **Invariant**: cached values MUST be strings whose `.length` equals the
|
|
54
|
+
* byte count we account for at insertion. We derive size on demand via
|
|
55
|
+
* `cache.get(key)?.length` rather than maintaining a parallel `Map<string, number>`.
|
|
56
|
+
* If you ever wrap the value (e.g. cache a Buffer or an object), the byte
|
|
57
|
+
* accounting silently breaks — switch to a parallel size map first.
|
|
58
|
+
*/
|
|
23
59
|
function createFrameSourceCache(
|
|
24
|
-
|
|
60
|
+
entryLimit: number,
|
|
61
|
+
bytesLimit: number,
|
|
25
62
|
frameSrcResolver?: (framePath: string) => string | null,
|
|
26
|
-
) {
|
|
63
|
+
): FrameSourceCache {
|
|
27
64
|
const cache = new Map<string, string>();
|
|
28
65
|
const inFlight = new Map<string, Promise<string>>();
|
|
66
|
+
let totalBytes = 0;
|
|
67
|
+
let evictions = 0;
|
|
68
|
+
let oversizedRejections = 0;
|
|
69
|
+
|
|
70
|
+
function evictOldest(): void {
|
|
71
|
+
const oldestKey = cache.keys().next().value;
|
|
72
|
+
if (!oldestKey) return;
|
|
73
|
+
// Snapshot the value before deleting so the byte-size derivation can't
|
|
74
|
+
// accidentally read post-delete (a future reorder would silently lose
|
|
75
|
+
// accounting and surface as `totalBytes` drifting out of sync).
|
|
76
|
+
const dropped = cache.get(oldestKey);
|
|
77
|
+
cache.delete(oldestKey);
|
|
78
|
+
totalBytes = Math.max(0, totalBytes - (dropped?.length ?? 0));
|
|
79
|
+
evictions++;
|
|
80
|
+
}
|
|
29
81
|
|
|
30
82
|
function remember(framePath: string, dataUri: string): string {
|
|
83
|
+
// Skip caching entries that alone exceed the byte budget. Caching them
|
|
84
|
+
// would trigger immediate self-eviction on insert and pollute LRU order
|
|
85
|
+
// by displacing the previous entry's slot.
|
|
86
|
+
if (dataUri.length > bytesLimit) {
|
|
87
|
+
oversizedRejections++;
|
|
88
|
+
// Drop any stale prior version so the caller sees consistent state.
|
|
89
|
+
if (cache.has(framePath)) {
|
|
90
|
+
totalBytes = Math.max(0, totalBytes - (cache.get(framePath)?.length ?? 0));
|
|
91
|
+
cache.delete(framePath);
|
|
92
|
+
}
|
|
93
|
+
return dataUri;
|
|
94
|
+
}
|
|
31
95
|
if (cache.has(framePath)) {
|
|
96
|
+
totalBytes = Math.max(0, totalBytes - (cache.get(framePath)?.length ?? 0));
|
|
32
97
|
cache.delete(framePath);
|
|
33
98
|
}
|
|
34
99
|
cache.set(framePath, dataUri);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
cache.delete(oldestKey);
|
|
39
|
-
}
|
|
100
|
+
totalBytes += dataUri.length;
|
|
101
|
+
while ((cache.size > entryLimit || totalBytes > bytesLimit) && cache.size > 0) {
|
|
102
|
+
evictOldest();
|
|
40
103
|
}
|
|
41
104
|
return dataUri;
|
|
42
105
|
}
|
|
@@ -70,9 +133,19 @@ function createFrameSourceCache(
|
|
|
70
133
|
return pending;
|
|
71
134
|
}
|
|
72
135
|
|
|
73
|
-
return {
|
|
136
|
+
return {
|
|
137
|
+
get,
|
|
138
|
+
stats: () => ({
|
|
139
|
+
entries: cache.size,
|
|
140
|
+
bytes: totalBytes,
|
|
141
|
+
evictions,
|
|
142
|
+
oversizedRejections,
|
|
143
|
+
}),
|
|
144
|
+
};
|
|
74
145
|
}
|
|
75
146
|
|
|
147
|
+
export const __testing = { createFrameSourceCache };
|
|
148
|
+
|
|
76
149
|
/**
|
|
77
150
|
* Creates a BeforeCaptureHook that injects pre-extracted video frames
|
|
78
151
|
* into the page, replacing native <video> elements with frame images.
|
|
@@ -83,11 +156,16 @@ export function createVideoFrameInjector(
|
|
|
83
156
|
): BeforeCaptureHook | null {
|
|
84
157
|
if (!frameLookup) return null;
|
|
85
158
|
|
|
86
|
-
const
|
|
159
|
+
const entryLimit = Math.max(
|
|
87
160
|
32,
|
|
88
161
|
config?.frameDataUriCacheLimit ?? DEFAULT_CONFIG.frameDataUriCacheLimit,
|
|
89
162
|
);
|
|
90
|
-
const
|
|
163
|
+
const bytesLimitMb = Math.max(
|
|
164
|
+
64,
|
|
165
|
+
config?.frameDataUriCacheBytesLimitMb ?? DEFAULT_CONFIG.frameDataUriCacheBytesLimitMb,
|
|
166
|
+
);
|
|
167
|
+
const bytesLimit = bytesLimitMb * 1024 * 1024;
|
|
168
|
+
const frameCache = createFrameSourceCache(entryLimit, bytesLimit, config?.frameSrcResolver);
|
|
91
169
|
const lastInjectedFrameByVideo = new Map<string, number>();
|
|
92
170
|
|
|
93
171
|
return async (page: Page, time: number) => {
|
|
@@ -511,6 +511,16 @@ describe("blitRgba8OverRgb48le", () => {
|
|
|
511
511
|
expect(canvas.readUInt16LE(4)).toBe(0);
|
|
512
512
|
});
|
|
513
513
|
|
|
514
|
+
it("fully opaque DOM with srgb transfer expands 8-bit channels to 16-bit SDR", () => {
|
|
515
|
+
const canvas = makeHdrFrame(1, 1, 10000, 20000, 30000);
|
|
516
|
+
const dom = makeDomRgba(1, 1, 255, 128, 1, 255);
|
|
517
|
+
blitRgba8OverRgb48le(dom, canvas, 1, 1, "srgb");
|
|
518
|
+
|
|
519
|
+
expect(canvas.readUInt16LE(0)).toBe(65535);
|
|
520
|
+
expect(canvas.readUInt16LE(2)).toBe(128 * 257);
|
|
521
|
+
expect(canvas.readUInt16LE(4)).toBe(257);
|
|
522
|
+
});
|
|
523
|
+
|
|
514
524
|
it("sRGB→HLG: black stays black, white stays white", () => {
|
|
515
525
|
const canvasBlack = makeHdrFrame(1, 1, 0, 0, 0);
|
|
516
526
|
const domBlack = makeDomRgba(1, 1, 0, 0, 0, 255);
|
package/src/utils/alphaBlit.ts
CHANGED
|
@@ -249,7 +249,7 @@ export function decodePngToRgb48le(buf: Buffer): { width: number; height: number
|
|
|
249
249
|
* bt2020). For neutral/near-neutral content (text, UI) the gamut difference
|
|
250
250
|
* is negligible.
|
|
251
251
|
*/
|
|
252
|
-
function
|
|
252
|
+
function buildSrgbToSignalLut(transfer: "hlg" | "pq" | "srgb"): Uint16Array {
|
|
253
253
|
const lut = new Uint16Array(256);
|
|
254
254
|
|
|
255
255
|
// HLG OETF constants (Rec. 2100)
|
|
@@ -267,6 +267,11 @@ function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
|
|
|
267
267
|
const sdrNits = 203.0;
|
|
268
268
|
|
|
269
269
|
for (let i = 0; i < 256; i++) {
|
|
270
|
+
if (transfer === "srgb") {
|
|
271
|
+
lut[i] = i * 257;
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
|
|
270
275
|
// sRGB EOTF: signal → linear (range 0–1, relative to SDR white)
|
|
271
276
|
const v = i / 255;
|
|
272
277
|
const linear = v <= 0.04045 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4);
|
|
@@ -288,12 +293,15 @@ function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
|
|
|
288
293
|
return lut;
|
|
289
294
|
}
|
|
290
295
|
|
|
291
|
-
const
|
|
292
|
-
const
|
|
296
|
+
const SRGB_TO_SRGB_16 = buildSrgbToSignalLut("srgb");
|
|
297
|
+
const SRGB_TO_HLG = buildSrgbToSignalLut("hlg");
|
|
298
|
+
const SRGB_TO_PQ = buildSrgbToSignalLut("pq");
|
|
293
299
|
|
|
294
300
|
/** Select the correct sRGB→HDR LUT for the given transfer function. */
|
|
295
|
-
function
|
|
296
|
-
|
|
301
|
+
function getSrgbToSignalLut(transfer: "hlg" | "pq" | "srgb"): Uint16Array {
|
|
302
|
+
if (transfer === "pq") return SRGB_TO_PQ;
|
|
303
|
+
if (transfer === "hlg") return SRGB_TO_HLG;
|
|
304
|
+
return SRGB_TO_SRGB_16;
|
|
297
305
|
}
|
|
298
306
|
|
|
299
307
|
// ── Alpha compositing ─────────────────────────────────────────────────────────
|
|
@@ -317,10 +325,10 @@ export function blitRgba8OverRgb48le(
|
|
|
317
325
|
canvas: Buffer,
|
|
318
326
|
width: number,
|
|
319
327
|
height: number,
|
|
320
|
-
transfer: "hlg" | "pq" = "hlg",
|
|
328
|
+
transfer: "hlg" | "pq" | "srgb" = "hlg",
|
|
321
329
|
): void {
|
|
322
330
|
const pixelCount = width * height;
|
|
323
|
-
const lut =
|
|
331
|
+
const lut = getSrgbToSignalLut(transfer);
|
|
324
332
|
|
|
325
333
|
for (let i = 0; i < pixelCount; i++) {
|
|
326
334
|
const da = domRgba[i * 4 + 3] ?? 0;
|
|
@@ -225,6 +225,46 @@ describe("ffprobe missing-binary fallback", () => {
|
|
|
225
225
|
expect(meta.hasAlpha).toBe(true);
|
|
226
226
|
});
|
|
227
227
|
|
|
228
|
+
// Regression: newer libavformat builds (and the output of `hyperframes
|
|
229
|
+
// remove-background` itself) write the VP9-alpha sidecar tag as
|
|
230
|
+
// `ALPHA_MODE` (uppercase). The lowercase-only check classified those
|
|
231
|
+
// files as having no alpha, the producer extracted them as JPGs, and
|
|
232
|
+
// the injected <img> overlays were fully opaque rectangles that hid
|
|
233
|
+
// every static element below them on the z-stack. The bug was silent —
|
|
234
|
+
// studio preview rendered correctly via native <video> playback while
|
|
235
|
+
// production renders covered headlines and captions with the avatar.
|
|
236
|
+
it("extractMediaMetadata detects ALPHA_MODE (uppercase) streams from newer ffmpeg builds", async () => {
|
|
237
|
+
const { spawn } = createSpawnSpy([
|
|
238
|
+
{
|
|
239
|
+
kind: "exit",
|
|
240
|
+
code: 0,
|
|
241
|
+
stdout: JSON.stringify({
|
|
242
|
+
streams: [
|
|
243
|
+
{
|
|
244
|
+
codec_type: "video",
|
|
245
|
+
codec_name: "vp9",
|
|
246
|
+
width: 320,
|
|
247
|
+
height: 180,
|
|
248
|
+
r_frame_rate: "30/1",
|
|
249
|
+
avg_frame_rate: "30/1",
|
|
250
|
+
pix_fmt: "yuv420p",
|
|
251
|
+
tags: { ALPHA_MODE: "1" },
|
|
252
|
+
},
|
|
253
|
+
],
|
|
254
|
+
format: { duration: "1.5" },
|
|
255
|
+
}),
|
|
256
|
+
},
|
|
257
|
+
]);
|
|
258
|
+
vi.resetModules();
|
|
259
|
+
vi.doMock("child_process", () => ({ spawn }));
|
|
260
|
+
|
|
261
|
+
const { extractMediaMetadata: extractMediaMetadataMocked } = await import("./ffprobe.js");
|
|
262
|
+
const meta = await extractMediaMetadataMocked("/tmp/alpha-uppercase.webm");
|
|
263
|
+
|
|
264
|
+
expect(meta.videoCodec).toBe("vp9");
|
|
265
|
+
expect(meta.hasAlpha).toBe(true);
|
|
266
|
+
});
|
|
267
|
+
|
|
228
268
|
it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
|
|
229
269
|
const { spawn } = createSpawnSpy([{ kind: "missing" }]);
|
|
230
270
|
vi.resetModules();
|
package/src/utils/ffprobe.ts
CHANGED
|
@@ -203,6 +203,21 @@ function extractStillImageMetadata(filePath: string): StillImageMetadata | null
|
|
|
203
203
|
}
|
|
204
204
|
}
|
|
205
205
|
|
|
206
|
+
/**
|
|
207
|
+
* Read an ffprobe tag case-insensitively. ffmpeg/libavformat versions disagree
|
|
208
|
+
* on tag casing — VP9 alpha is `alpha_mode` in older builds and `ALPHA_MODE`
|
|
209
|
+
* in newer ones; HDR tags vary similarly. Use this for any sidecar tag where
|
|
210
|
+
* you want to be resilient across muxer versions.
|
|
211
|
+
*/
|
|
212
|
+
function readTagCI(tags: Record<string, string | undefined> | undefined, name: string): string {
|
|
213
|
+
if (!tags) return "";
|
|
214
|
+
const target = name.toLowerCase();
|
|
215
|
+
for (const [key, value] of Object.entries(tags)) {
|
|
216
|
+
if (key.toLowerCase() === target && typeof value === "string") return value;
|
|
217
|
+
}
|
|
218
|
+
return "";
|
|
219
|
+
}
|
|
220
|
+
|
|
206
221
|
function parseFrameRate(frameRateStr: string | undefined): number {
|
|
207
222
|
if (!frameRateStr) return 0;
|
|
208
223
|
const parts = frameRateStr.split("/");
|
|
@@ -277,7 +292,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
|
|
|
277
292
|
: null;
|
|
278
293
|
const colorSpace = ffprobeColorSpace ?? stillImageMeta?.colorSpace ?? null;
|
|
279
294
|
const pixelFormat = videoStream.pix_fmt || "";
|
|
280
|
-
const alphaMode = videoStream.tags
|
|
295
|
+
const alphaMode = readTagCI(videoStream.tags, "alpha_mode");
|
|
281
296
|
const hasAlpha =
|
|
282
297
|
/(^|[^a-z])yuva|rgba|argb|bgra|gbrap|gray[a-z0-9]*a/i.test(pixelFormat) || alphaMode === "1";
|
|
283
298
|
|