@hyperframes/engine 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +9 -0
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/services/browserManager.d.ts.map +1 -1
- package/dist/services/browserManager.js +6 -3
- package/dist/services/browserManager.js.map +1 -1
- package/dist/services/chunkEncoder.d.ts +14 -7
- package/dist/services/chunkEncoder.d.ts.map +1 -1
- package/dist/services/chunkEncoder.js +25 -9
- package/dist/services/chunkEncoder.js.map +1 -1
- package/dist/services/chunkEncoder.types.d.ts +4 -0
- package/dist/services/chunkEncoder.types.d.ts.map +1 -1
- package/dist/services/frameCapture.d.ts.map +1 -1
- package/dist/services/frameCapture.js +18 -2
- package/dist/services/frameCapture.js.map +1 -1
- package/dist/services/hdrCapture.d.ts +62 -0
- package/dist/services/hdrCapture.d.ts.map +1 -0
- package/dist/services/hdrCapture.js +259 -0
- package/dist/services/hdrCapture.js.map +1 -0
- package/dist/services/screenshotService.d.ts +34 -0
- package/dist/services/screenshotService.d.ts.map +1 -1
- package/dist/services/screenshotService.js +97 -19
- package/dist/services/screenshotService.js.map +1 -1
- package/dist/services/streamingEncoder.d.ts +18 -3
- package/dist/services/streamingEncoder.d.ts.map +1 -1
- package/dist/services/streamingEncoder.js +104 -50
- package/dist/services/streamingEncoder.js.map +1 -1
- package/dist/services/videoFrameExtractor.d.ts.map +1 -1
- package/dist/services/videoFrameExtractor.js +109 -18
- package/dist/services/videoFrameExtractor.js.map +1 -1
- package/dist/services/videoFrameInjector.d.ts +54 -0
- package/dist/services/videoFrameInjector.d.ts.map +1 -1
- package/dist/services/videoFrameInjector.js +159 -0
- package/dist/services/videoFrameInjector.js.map +1 -1
- package/dist/utils/alphaBlit.d.ts +105 -0
- package/dist/utils/alphaBlit.d.ts.map +1 -0
- package/dist/utils/alphaBlit.js +550 -0
- package/dist/utils/alphaBlit.js.map +1 -0
- package/dist/utils/ffprobe.d.ts +10 -0
- package/dist/utils/ffprobe.d.ts.map +1 -1
- package/dist/utils/ffprobe.js +7 -0
- package/dist/utils/ffprobe.js.map +1 -1
- package/dist/utils/hdr.d.ts +82 -0
- package/dist/utils/hdr.d.ts.map +1 -0
- package/dist/utils/hdr.js +87 -0
- package/dist/utils/hdr.js.map +1 -0
- package/dist/utils/layerCompositor.d.ts +36 -0
- package/dist/utils/layerCompositor.d.ts.map +1 -0
- package/dist/utils/layerCompositor.js +49 -0
- package/dist/utils/layerCompositor.js.map +1 -0
- package/package.json +3 -2
- package/src/config.ts +16 -0
- package/src/index.ts +42 -0
- package/src/services/browserManager.ts +6 -3
- package/src/services/chunkEncoder.test.ts +88 -0
- package/src/services/chunkEncoder.ts +35 -9
- package/src/services/chunkEncoder.types.ts +3 -0
- package/src/services/frameCapture.ts +31 -4
- package/src/services/hdrCapture.test.ts +159 -0
- package/src/services/hdrCapture.ts +354 -0
- package/src/services/screenshotService.ts +102 -17
- package/src/services/streamingEncoder.test.ts +228 -0
- package/src/services/streamingEncoder.ts +153 -63
- package/src/services/videoFrameExtractor.ts +135 -31
- package/src/services/videoFrameInjector.ts +200 -0
- package/src/utils/alphaBlit.test.ts +993 -0
- package/src/utils/alphaBlit.ts +643 -0
- package/src/utils/ffprobe.ts +22 -0
- package/src/utils/hdr.test.ts +191 -0
- package/src/utils/hdr.ts +137 -0
- package/src/utils/layerCompositor.test.ts +141 -0
- package/src/utils/layerCompositor.ts +58 -0
- package/tsconfig.json +2 -1
|
@@ -133,6 +133,82 @@ export async function pageScreenshotCapture(page: Page, options: CaptureOptions)
|
|
|
133
133
|
return Buffer.from(result.data, "base64");
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
/**
|
|
137
|
+
* Capture a screenshot with transparent background (PNG + alpha channel).
|
|
138
|
+
*
|
|
139
|
+
* Used in the two-pass HDR compositing pipeline — captures DOM content
|
|
140
|
+
* (text, graphics, SDR overlays) with transparency where the background shows,
|
|
141
|
+
* so it can be overlaid on top of native HDR video frames in FFmpeg.
|
|
142
|
+
*
|
|
143
|
+
* Sets and restores the background color override on every call. For sessions
|
|
144
|
+
* that capture many frames, prefer calling initTransparentBackground() once
|
|
145
|
+
* at session init, then captureAlphaPng() per frame to avoid the 2× CDP
|
|
146
|
+
* round-trip overhead.
|
|
147
|
+
*/
|
|
148
|
+
export async function captureScreenshotWithAlpha(
|
|
149
|
+
page: Page,
|
|
150
|
+
width: number,
|
|
151
|
+
height: number,
|
|
152
|
+
): Promise<Buffer> {
|
|
153
|
+
const client = await getCdpSession(page);
|
|
154
|
+
// Force transparent background so the screenshot has a real alpha channel
|
|
155
|
+
await client.send("Emulation.setDefaultBackgroundColorOverride", {
|
|
156
|
+
color: { r: 0, g: 0, b: 0, a: 0 },
|
|
157
|
+
});
|
|
158
|
+
try {
|
|
159
|
+
const result = await client.send("Page.captureScreenshot", {
|
|
160
|
+
format: "png",
|
|
161
|
+
fromSurface: true,
|
|
162
|
+
captureBeyondViewport: false,
|
|
163
|
+
optimizeForSpeed: false, // `true` uses a zero-alpha-aware fast path that crushes real alpha values — observed empirically, CDP docs don't spell it out
|
|
164
|
+
clip: { x: 0, y: 0, width, height, scale: 1 },
|
|
165
|
+
});
|
|
166
|
+
return Buffer.from(result.data, "base64");
|
|
167
|
+
} finally {
|
|
168
|
+
// Restore opaque background even if captureScreenshot throws, otherwise
|
|
169
|
+
// subsequent opaque captures keep a transparent background.
|
|
170
|
+
await client.send("Emulation.setDefaultBackgroundColorOverride", {}).catch(() => {});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Set the page background to transparent once for a dedicated HDR DOM session.
|
|
176
|
+
*
|
|
177
|
+
* Call this once after session initialization. Then use captureAlphaPng() per
|
|
178
|
+
* frame instead of captureScreenshotWithAlpha() to skip the per-frame CDP
|
|
179
|
+
* background override round-trips.
|
|
180
|
+
*
|
|
181
|
+
* Only use on sessions that are exclusively dedicated to transparent capture
|
|
182
|
+
* (e.g., the HDR two-pass DOM layer session) — the background will stay
|
|
183
|
+
* transparent for the lifetime of the session.
|
|
184
|
+
*/
|
|
185
|
+
export async function initTransparentBackground(page: Page): Promise<void> {
|
|
186
|
+
const client = await getCdpSession(page);
|
|
187
|
+
await client.send("Emulation.setDefaultBackgroundColorOverride", {
|
|
188
|
+
color: { r: 0, g: 0, b: 0, a: 0 },
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Capture a transparent-background PNG screenshot without setting the
|
|
194
|
+
* background color override. Requires initTransparentBackground() to have
|
|
195
|
+
* been called once on this session.
|
|
196
|
+
*
|
|
197
|
+
* Faster than captureScreenshotWithAlpha() for per-frame use in the HDR
|
|
198
|
+
* two-pass compositing loop.
|
|
199
|
+
*/
|
|
200
|
+
export async function captureAlphaPng(page: Page, width: number, height: number): Promise<Buffer> {
|
|
201
|
+
const client = await getCdpSession(page);
|
|
202
|
+
const result = await client.send("Page.captureScreenshot", {
|
|
203
|
+
format: "png",
|
|
204
|
+
fromSurface: true,
|
|
205
|
+
captureBeyondViewport: false,
|
|
206
|
+
optimizeForSpeed: false, // must be false to preserve alpha
|
|
207
|
+
clip: { x: 0, y: 0, width, height, scale: 1 },
|
|
208
|
+
});
|
|
209
|
+
return Buffer.from(result.data, "base64");
|
|
210
|
+
}
|
|
211
|
+
|
|
136
212
|
export async function injectVideoFramesBatch(
|
|
137
213
|
page: Page,
|
|
138
214
|
updates: Array<{ videoId: string; dataUri: string }>,
|
|
@@ -160,16 +236,11 @@ export async function injectVideoFramesBatch(
|
|
|
160
236
|
}
|
|
161
237
|
if (!img) continue;
|
|
162
238
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
img.style.left = computedStyle.left;
|
|
169
|
-
img.style.right = computedStyle.right;
|
|
170
|
-
img.style.bottom = computedStyle.bottom;
|
|
171
|
-
img.style.inset = computedStyle.inset;
|
|
172
|
-
} else {
|
|
239
|
+
// Always use absolute positioning so the <img> overlays the <video>
|
|
240
|
+
// instead of flowing below it. With position:relative, both elements
|
|
241
|
+
// stack vertically — the <img> lands below the video and gets clipped
|
|
242
|
+
// by any overflow:hidden ancestor (e.g., border-radius wrappers).
|
|
243
|
+
{
|
|
173
244
|
const videoRect = video.getBoundingClientRect();
|
|
174
245
|
const offsetLeft = Number.isFinite(video.offsetLeft) ? video.offsetLeft : 0;
|
|
175
246
|
const offsetTop = Number.isFinite(video.offsetTop) ? video.offsetTop : 0;
|
|
@@ -235,14 +306,28 @@ export async function syncVideoFrameVisibility(
|
|
|
235
306
|
const active = new Set(ids);
|
|
236
307
|
const videos = Array.from(document.querySelectorAll("video[data-start]")) as HTMLVideoElement[];
|
|
237
308
|
for (const video of videos) {
|
|
238
|
-
if (active.has(video.id)) continue;
|
|
239
|
-
video.style.removeProperty("display");
|
|
240
|
-
video.style.setProperty("visibility", "hidden", "important");
|
|
241
|
-
video.style.setProperty("opacity", "0", "important");
|
|
242
|
-
video.style.setProperty("pointer-events", "none", "important");
|
|
243
309
|
const img = video.nextElementSibling as HTMLElement | null;
|
|
244
|
-
|
|
245
|
-
|
|
310
|
+
const hasImg = img && img.classList.contains("__render_frame__");
|
|
311
|
+
if (active.has(video.id)) {
|
|
312
|
+
// Active video: show injected <img>, hide native <video>.
|
|
313
|
+
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
|
|
314
|
+
// survive until injectVideoFramesBatch reads it via getComputedStyle.
|
|
315
|
+
// visibility:hidden alone hides the native element without affecting
|
|
316
|
+
// its computed opacity.
|
|
317
|
+
video.style.setProperty("visibility", "hidden", "important");
|
|
318
|
+
video.style.setProperty("pointer-events", "none", "important");
|
|
319
|
+
if (hasImg) {
|
|
320
|
+
img.style.visibility = "visible";
|
|
321
|
+
}
|
|
322
|
+
} else {
|
|
323
|
+
// Inactive video: hide both
|
|
324
|
+
video.style.removeProperty("display");
|
|
325
|
+
video.style.setProperty("visibility", "hidden", "important");
|
|
326
|
+
video.style.setProperty("opacity", "0", "important");
|
|
327
|
+
video.style.setProperty("pointer-events", "none", "important");
|
|
328
|
+
if (hasImg) {
|
|
329
|
+
img.style.visibility = "hidden";
|
|
330
|
+
}
|
|
246
331
|
}
|
|
247
332
|
}
|
|
248
333
|
}, activeVideoIds);
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* buildStreamingArgs unit tests.
|
|
3
|
+
*
|
|
4
|
+
* These tests focus on the FFmpeg CLI shape rather than spawning the encoder
|
|
5
|
+
* — they're the cheap regression net for the HDR static-metadata bug
|
|
6
|
+
* (side_data=[none] in the encoded MP4) reproduced by
|
|
7
|
+
* packages/producer/scripts/hdr-smoke.ts. Without these assertions, future
|
|
8
|
+
* refactors of the x265-params string can silently strip
|
|
9
|
+
* master-display / max-cll and ship as SDR BT.2020 again.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { describe, expect, it } from "vitest";
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
buildStreamingArgs,
|
|
16
|
+
createFrameReorderBuffer,
|
|
17
|
+
type StreamingEncoderOptions,
|
|
18
|
+
} from "./streamingEncoder.js";
|
|
19
|
+
import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js";
|
|
20
|
+
|
|
21
|
+
const baseHdrPq: StreamingEncoderOptions = {
|
|
22
|
+
fps: 30,
|
|
23
|
+
width: 1920,
|
|
24
|
+
height: 1080,
|
|
25
|
+
codec: "h265",
|
|
26
|
+
preset: "medium",
|
|
27
|
+
quality: 23,
|
|
28
|
+
pixelFormat: "yuv420p10le",
|
|
29
|
+
useGpu: false,
|
|
30
|
+
rawInputFormat: "rgb48le",
|
|
31
|
+
hdr: { transfer: "pq" },
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const baseHdrHlg: StreamingEncoderOptions = {
|
|
35
|
+
...baseHdrPq,
|
|
36
|
+
hdr: { transfer: "hlg" },
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const baseSdr: StreamingEncoderOptions = {
|
|
40
|
+
fps: 30,
|
|
41
|
+
width: 1920,
|
|
42
|
+
height: 1080,
|
|
43
|
+
codec: "h264",
|
|
44
|
+
preset: "medium",
|
|
45
|
+
quality: 23,
|
|
46
|
+
useGpu: false,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
function getX265ParamsValue(args: string[]): string | undefined {
|
|
50
|
+
const idx = args.indexOf("-x265-params");
|
|
51
|
+
return idx === -1 ? undefined : args[idx + 1];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
describe("buildStreamingArgs", () => {
|
|
55
|
+
describe("HDR PQ (libx265)", () => {
|
|
56
|
+
it("emits master-display and max-cll in -x265-params", () => {
|
|
57
|
+
const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
|
|
58
|
+
const x265 = getX265ParamsValue(args);
|
|
59
|
+
expect(x265).toBeDefined();
|
|
60
|
+
expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
|
|
61
|
+
expect(x265).toContain(`max-cll=${DEFAULT_HDR10_MASTERING.maxCll}`);
|
|
62
|
+
expect(x265).toContain("colorprim=bt2020");
|
|
63
|
+
expect(x265).toContain("transfer=smpte2084");
|
|
64
|
+
expect(x265).toContain("colormatrix=bt2020nc");
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("tags the output stream with bt2020 / smpte2084 / tv range", () => {
|
|
68
|
+
const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
|
|
69
|
+
expect(args).toContain("-colorspace:v");
|
|
70
|
+
expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt2020nc");
|
|
71
|
+
expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt2020");
|
|
72
|
+
expect(args[args.indexOf("-color_trc:v") + 1]).toBe("smpte2084");
|
|
73
|
+
expect(args[args.indexOf("-color_range") + 1]).toBe("tv");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("uses libx265 with -tag:v hvc1 for QuickTime compatibility", () => {
|
|
77
|
+
const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
|
|
78
|
+
const cvIdx = args.indexOf("-c:v");
|
|
79
|
+
expect(cvIdx).toBeGreaterThan(-1);
|
|
80
|
+
expect(args[cvIdx + 1]).toBe("libx265");
|
|
81
|
+
expect(args).toContain("-tag:v");
|
|
82
|
+
expect(args[args.indexOf("-tag:v") + 1]).toBe("hvc1");
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("keeps the aq-mode prefix even with master-display present", () => {
|
|
86
|
+
const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
|
|
87
|
+
const x265 = getX265ParamsValue(args);
|
|
88
|
+
expect(x265?.startsWith("aq-mode=3")).toBe(true);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it("uses the simpler aq-mode prefix on ultrafast preset", () => {
|
|
92
|
+
const args = buildStreamingArgs({ ...baseHdrPq, preset: "ultrafast" }, "/tmp/out.mp4");
|
|
93
|
+
const x265 = getX265ParamsValue(args);
|
|
94
|
+
expect(x265?.startsWith("aq-mode=3:")).toBe(true);
|
|
95
|
+
expect(x265).not.toContain("aq-strength");
|
|
96
|
+
expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
describe("HDR HLG (libx265)", () => {
|
|
101
|
+
it("emits master-display, max-cll, and the HLG transfer", () => {
|
|
102
|
+
const args = buildStreamingArgs(baseHdrHlg, "/tmp/out.mp4");
|
|
103
|
+
const x265 = getX265ParamsValue(args);
|
|
104
|
+
expect(x265).toContain("transfer=arib-std-b67");
|
|
105
|
+
expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
|
|
106
|
+
expect(x265).toContain(`max-cll=${DEFAULT_HDR10_MASTERING.maxCll}`);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("tags the output stream with arib-std-b67", () => {
|
|
110
|
+
const args = buildStreamingArgs(baseHdrHlg, "/tmp/out.mp4");
|
|
111
|
+
expect(args[args.indexOf("-color_trc:v") + 1]).toBe("arib-std-b67");
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe("HDR raw input tagging", () => {
|
|
116
|
+
it("tags the rawvideo input with the matching color metadata", () => {
|
|
117
|
+
const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
|
|
118
|
+
const inputColorTrcIdx = args.indexOf("-color_trc");
|
|
119
|
+
expect(inputColorTrcIdx).toBeGreaterThan(-1);
|
|
120
|
+
expect(args[inputColorTrcIdx + 1]).toBe("smpte2084");
|
|
121
|
+
const inputPrimariesIdx = args.indexOf("-color_primaries");
|
|
122
|
+
expect(inputPrimariesIdx).toBeGreaterThan(-1);
|
|
123
|
+
expect(args[inputPrimariesIdx + 1]).toBe("bt2020");
|
|
124
|
+
// Pix_fmt of the raw input must match the buffer we hand FFmpeg.
|
|
125
|
+
expect(args.indexOf("rgb48le")).toBeGreaterThan(-1);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("does not strip the input color tags when bitrate is set instead of CRF", () => {
|
|
129
|
+
const args = buildStreamingArgs({ ...baseHdrPq, bitrate: "20M" }, "/tmp/out.mp4");
|
|
130
|
+
const x265 = getX265ParamsValue(args);
|
|
131
|
+
expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
|
|
132
|
+
expect(args).toContain("-b:v");
|
|
133
|
+
expect(args[args.indexOf("-b:v") + 1]).toBe("20M");
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("SDR fallback", () => {
|
|
138
|
+
it("does NOT emit HDR mastering metadata for SDR encodes", () => {
|
|
139
|
+
const args = buildStreamingArgs(baseSdr, "/tmp/out.mp4");
|
|
140
|
+
const x264 = args[args.indexOf("-x264-params") + 1];
|
|
141
|
+
expect(x264).toContain("colorprim=bt709");
|
|
142
|
+
expect(x264).toContain("transfer=bt709");
|
|
143
|
+
expect(x264).toContain("colormatrix=bt709");
|
|
144
|
+
expect(x264).not.toContain("master-display");
|
|
145
|
+
expect(x264).not.toContain("max-cll");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it("tags SDR output with bt709 and tv range", () => {
|
|
149
|
+
const args = buildStreamingArgs(baseSdr, "/tmp/out.mp4");
|
|
150
|
+
expect(args[args.indexOf("-color_trc:v") + 1]).toBe("bt709");
|
|
151
|
+
expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt709");
|
|
152
|
+
expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt709");
|
|
153
|
+
expect(args[args.indexOf("-color_range") + 1]).toBe("tv");
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
describe("output path", () => {
|
|
158
|
+
it("places the output path last after -y", () => {
|
|
159
|
+
const args = buildStreamingArgs(baseHdrPq, "/tmp/some-output.mp4");
|
|
160
|
+
expect(args[args.length - 2]).toBe("-y");
|
|
161
|
+
expect(args[args.length - 1]).toBe("/tmp/some-output.mp4");
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
describe("createFrameReorderBuffer", () => {
|
|
167
|
+
it("fast-paths waitForFrame(cursor) without queueing", async () => {
|
|
168
|
+
const buf = createFrameReorderBuffer(0, 3);
|
|
169
|
+
await buf.waitForFrame(0);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("gates out-of-order writers into cursor order", async () => {
|
|
173
|
+
const buf = createFrameReorderBuffer(0, 4);
|
|
174
|
+
const writeOrder: number[] = [];
|
|
175
|
+
|
|
176
|
+
const writer = async (frame: number) => {
|
|
177
|
+
await buf.waitForFrame(frame);
|
|
178
|
+
writeOrder.push(frame);
|
|
179
|
+
buf.advanceTo(frame + 1);
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
const p3 = writer(3);
|
|
183
|
+
const p1 = writer(1);
|
|
184
|
+
const p2 = writer(2);
|
|
185
|
+
const p0 = writer(0);
|
|
186
|
+
|
|
187
|
+
await Promise.all([p0, p1, p2, p3]);
|
|
188
|
+
expect(writeOrder).toEqual([0, 1, 2, 3]);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
it("supports multiple waiters registered for the same frame", async () => {
|
|
192
|
+
const buf = createFrameReorderBuffer(0, 2);
|
|
193
|
+
const resolved: string[] = [];
|
|
194
|
+
|
|
195
|
+
const a = buf.waitForFrame(1).then(() => resolved.push("a"));
|
|
196
|
+
const b = buf.waitForFrame(1).then(() => resolved.push("b"));
|
|
197
|
+
|
|
198
|
+
buf.advanceTo(0);
|
|
199
|
+
await Promise.resolve();
|
|
200
|
+
expect(resolved).toEqual([]);
|
|
201
|
+
|
|
202
|
+
buf.advanceTo(1);
|
|
203
|
+
await Promise.all([a, b]);
|
|
204
|
+
expect(resolved.sort()).toEqual(["a", "b"]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("waitForAllDone resolves when cursor reaches endFrame", async () => {
|
|
208
|
+
const buf = createFrameReorderBuffer(0, 3);
|
|
209
|
+
let done = false;
|
|
210
|
+
const allDone = buf.waitForAllDone().then(() => {
|
|
211
|
+
done = true;
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
buf.advanceTo(1);
|
|
215
|
+
await Promise.resolve();
|
|
216
|
+
expect(done).toBe(false);
|
|
217
|
+
|
|
218
|
+
buf.advanceTo(3);
|
|
219
|
+
await allDone;
|
|
220
|
+
expect(done).toBe(true);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("waitForAllDone fast-paths when cursor already past endFrame", async () => {
|
|
224
|
+
const buf = createFrameReorderBuffer(0, 3);
|
|
225
|
+
buf.advanceTo(5);
|
|
226
|
+
await buf.waitForAllDone();
|
|
227
|
+
});
|
|
228
|
+
});
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Streaming Encoder Service
|
|
3
3
|
*
|
|
4
|
-
* Pipes frame screenshot buffers directly to FFmpeg's stdin
|
|
5
|
-
* them to disk and reading them back in a separate encode
|
|
6
|
-
* Remotion
|
|
4
|
+
* Pipes frame screenshot buffers directly to FFmpeg's stdin via `-f image2pipe`
|
|
5
|
+
* instead of writing them to disk and reading them back in a separate encode
|
|
6
|
+
* stage. Inspired by Remotion's approach to browser-based video rendering.
|
|
7
7
|
*
|
|
8
8
|
* Two building blocks:
|
|
9
9
|
* 1. Frame reorder buffer – ensures out-of-order parallel workers feed
|
|
@@ -17,6 +17,7 @@ import { existsSync, mkdirSync, statSync } from "fs";
|
|
|
17
17
|
import { dirname } from "path";
|
|
18
18
|
|
|
19
19
|
import { type GpuEncoder, getCachedGpuEncoder, getGpuEncoderName } from "../utils/gpuEncoder.js";
|
|
20
|
+
import { getHdrEncoderColorParams } from "../utils/hdr.js";
|
|
20
21
|
import { type EncoderOptions } from "./chunkEncoder.types.js";
|
|
21
22
|
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
|
22
23
|
|
|
@@ -24,8 +25,16 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
|
|
|
24
25
|
export type { EncoderOptions } from "./chunkEncoder.types.js";
|
|
25
26
|
|
|
26
27
|
// ---------------------------------------------------------------------------
|
|
27
|
-
// 1. Frame reorder buffer
|
|
28
|
+
// 1. Frame reorder buffer — ordered async barrier
|
|
28
29
|
// ---------------------------------------------------------------------------
|
|
30
|
+
//
|
|
31
|
+
// Parallel workers produce frames out of order; FFmpeg's stdin expects them in
|
|
32
|
+
// strict sequential order. Each worker calls `waitForFrame(n)` to block until
|
|
33
|
+
// its turn, writes, then calls `advanceTo(n + 1)` to release the next waiter.
|
|
34
|
+
//
|
|
35
|
+
// `pending` holds an array per frame index (not a single resolver) so that
|
|
36
|
+
// `waitForAllDone` can coexist with the writer still waiting on the final
|
|
37
|
+
// frame without one clobbering the other.
|
|
29
38
|
|
|
30
39
|
export interface FrameReorderBuffer {
|
|
31
40
|
waitForFrame: (frame: number) => Promise<void>;
|
|
@@ -34,34 +43,49 @@ export interface FrameReorderBuffer {
|
|
|
34
43
|
}
|
|
35
44
|
|
|
36
45
|
export function createFrameReorderBuffer(startFrame: number, endFrame: number): FrameReorderBuffer {
|
|
37
|
-
let
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
let cursor = startFrame;
|
|
47
|
+
const pending = new Map<number, Array<() => void>>();
|
|
48
|
+
|
|
49
|
+
const enqueueAt = (frame: number, resolve: () => void): void => {
|
|
50
|
+
const list = pending.get(frame);
|
|
51
|
+
if (list === undefined) {
|
|
52
|
+
pending.set(frame, [resolve]);
|
|
53
|
+
} else {
|
|
54
|
+
list.push(resolve);
|
|
46
55
|
}
|
|
47
56
|
};
|
|
48
57
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}),
|
|
55
|
-
advanceTo: (frame: number) => {
|
|
56
|
-
nextFrame = frame;
|
|
57
|
-
resolveWaiters();
|
|
58
|
-
},
|
|
59
|
-
waitForAllDone: () =>
|
|
60
|
-
new Promise<void>((resolve) => {
|
|
61
|
-
waiters.push({ frame: endFrame, resolve });
|
|
62
|
-
resolveWaiters();
|
|
63
|
-
}),
|
|
58
|
+
const flushAt = (frame: number): void => {
|
|
59
|
+
const list = pending.get(frame);
|
|
60
|
+
if (list === undefined) return;
|
|
61
|
+
pending.delete(frame);
|
|
62
|
+
for (const resolve of list) resolve();
|
|
64
63
|
};
|
|
64
|
+
|
|
65
|
+
const waitForFrame = (frame: number): Promise<void> =>
|
|
66
|
+
new Promise<void>((resolve) => {
|
|
67
|
+
if (frame === cursor) {
|
|
68
|
+
resolve();
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
enqueueAt(frame, resolve);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const advanceTo = (frame: number): void => {
|
|
75
|
+
cursor = frame;
|
|
76
|
+
flushAt(frame);
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const waitForAllDone = (): Promise<void> =>
|
|
80
|
+
new Promise<void>((resolve) => {
|
|
81
|
+
if (cursor >= endFrame) {
|
|
82
|
+
resolve();
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
enqueueAt(endFrame, resolve);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
return { waitForFrame, advanceTo, waitForAllDone };
|
|
65
89
|
}
|
|
66
90
|
|
|
67
91
|
// ---------------------------------------------------------------------------
|
|
@@ -79,6 +103,9 @@ export interface StreamingEncoderOptions {
|
|
|
79
103
|
pixelFormat?: string;
|
|
80
104
|
useGpu?: boolean;
|
|
81
105
|
imageFormat?: "jpeg" | "png";
|
|
106
|
+
hdr?: { transfer: import("../utils/hdr.js").HdrTransfer };
|
|
107
|
+
/** When set, use rawvideo input instead of image2pipe. For HDR PQ-encoded frames. */
|
|
108
|
+
rawInputFormat?: "rgb48le";
|
|
82
109
|
}
|
|
83
110
|
|
|
84
111
|
export interface StreamingEncoderResult {
|
|
@@ -98,8 +125,11 @@ export interface StreamingEncoder {
|
|
|
98
125
|
* Build FFmpeg args for streaming (image2pipe) input.
|
|
99
126
|
* Reuses the same codec/quality/GPU logic as chunkEncoder's buildEncoderArgs
|
|
100
127
|
* but with `-f image2pipe` instead of `-i <pattern>`.
|
|
128
|
+
*
|
|
129
|
+
* Exported so unit tests can assert on the constructed CLI without spawning
|
|
130
|
+
* FFmpeg — see streamingEncoder.test.ts.
|
|
101
131
|
*/
|
|
102
|
-
function buildStreamingArgs(
|
|
132
|
+
export function buildStreamingArgs(
|
|
103
133
|
options: StreamingEncoderOptions,
|
|
104
134
|
outputPath: string,
|
|
105
135
|
gpuEncoder: GpuEncoder = null,
|
|
@@ -116,19 +146,41 @@ function buildStreamingArgs(
|
|
|
116
146
|
} = options;
|
|
117
147
|
|
|
118
148
|
// Input args: pipe from stdin
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
149
|
+
const args: string[] = [];
|
|
150
|
+
if (options.rawInputFormat) {
|
|
151
|
+
// Raw pixel input (HLG/PQ-encoded rgb48le from FFmpeg extraction).
|
|
152
|
+
// Tag the input with the correct color space so FFmpeg uses the right
|
|
153
|
+
// YUV matrix when converting rgb48le → yuv420p10le for encoding.
|
|
154
|
+
// Without these tags FFmpeg assumes bt709 and applies the wrong matrix.
|
|
155
|
+
const hdrTransfer = options.hdr?.transfer;
|
|
156
|
+
const inputColorTrc =
|
|
157
|
+
hdrTransfer === "pq" ? "smpte2084" : hdrTransfer === "hlg" ? "arib-std-b67" : undefined;
|
|
158
|
+
args.push(
|
|
159
|
+
"-f",
|
|
160
|
+
"rawvideo",
|
|
161
|
+
"-pix_fmt",
|
|
162
|
+
options.rawInputFormat,
|
|
163
|
+
"-s",
|
|
164
|
+
`${options.width}x${options.height}`,
|
|
165
|
+
"-framerate",
|
|
166
|
+
String(fps),
|
|
167
|
+
);
|
|
168
|
+
if (inputColorTrc) {
|
|
169
|
+
args.push(
|
|
170
|
+
"-color_primaries",
|
|
171
|
+
"bt2020",
|
|
172
|
+
"-color_trc",
|
|
173
|
+
inputColorTrc,
|
|
174
|
+
"-colorspace",
|
|
175
|
+
"bt2020nc",
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
args.push("-i", "-");
|
|
179
|
+
} else {
|
|
180
|
+
const inputCodec = imageFormat === "png" ? "png" : "mjpeg";
|
|
181
|
+
args.push("-f", "image2pipe", "-vcodec", inputCodec, "-framerate", String(fps), "-i", "-");
|
|
182
|
+
}
|
|
183
|
+
args.push("-r", String(fps));
|
|
132
184
|
|
|
133
185
|
const shouldUseGpu = useGpu && gpuEncoder !== null;
|
|
134
186
|
|
|
@@ -169,16 +221,25 @@ function buildStreamingArgs(
|
|
|
169
221
|
if (bitrate) args.push("-b:v", bitrate);
|
|
170
222
|
else args.push("-crf", String(quality));
|
|
171
223
|
|
|
172
|
-
// Encoder-specific params: anti-banding +
|
|
173
|
-
//
|
|
174
|
-
//
|
|
224
|
+
// Encoder-specific params: anti-banding + color space tagging.
|
|
225
|
+
// For HDR, getHdrEncoderColorParams also emits the SMPTE ST 2086
|
|
226
|
+
// mastering-display and CTA-861.3 MaxCLL/MaxFALL SEI messages —
|
|
227
|
+
// without them, players (Apple, YouTube, HDR TVs) treat the file
|
|
228
|
+
// as SDR BT.2020 and tone-map incorrectly.
|
|
175
229
|
const xParamsFlag = codec === "h264" ? "-x264-params" : "-x265-params";
|
|
176
|
-
const colorParams =
|
|
230
|
+
const colorParams =
|
|
231
|
+
options.rawInputFormat && options.hdr
|
|
232
|
+
? getHdrEncoderColorParams(options.hdr.transfer).x265ColorParams
|
|
233
|
+
: "colorprim=bt709:transfer=bt709:colormatrix=bt709";
|
|
177
234
|
if (preset === "ultrafast") {
|
|
178
235
|
args.push(xParamsFlag, `aq-mode=3:${colorParams}`);
|
|
179
236
|
} else {
|
|
180
237
|
args.push(xParamsFlag, `aq-mode=3:aq-strength=0.8:deblock=1,1:${colorParams}`);
|
|
181
238
|
}
|
|
239
|
+
// Apple devices require hvc1 tag for HEVC playback (default hev1 won't open in QuickTime)
|
|
240
|
+
if (codec === "h265") {
|
|
241
|
+
args.push("-tag:v", "hvc1");
|
|
242
|
+
}
|
|
182
243
|
}
|
|
183
244
|
} else if (codec === "vp9") {
|
|
184
245
|
args.push("-c:v", "libvpx-vp9", "-b:v", bitrate || "0", "-crf", String(quality));
|
|
@@ -194,27 +255,47 @@ function buildStreamingArgs(
|
|
|
194
255
|
return [...args, "-y", outputPath];
|
|
195
256
|
}
|
|
196
257
|
|
|
197
|
-
//
|
|
198
|
-
//
|
|
258
|
+
// Color space metadata.
|
|
259
|
+
// When rawInputFormat is set, data comes from the WebGPU HDR pipeline
|
|
260
|
+
// (PQ-encoded) — tag with bt2020/PQ truthfully.
|
|
261
|
+
// Otherwise, Chrome captures sRGB — tag as bt709.
|
|
199
262
|
if (codec === "h264" || codec === "h265") {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
263
|
+
if (options.rawInputFormat && options.hdr) {
|
|
264
|
+
args.push(
|
|
265
|
+
"-colorspace:v",
|
|
266
|
+
"bt2020nc",
|
|
267
|
+
"-color_primaries:v",
|
|
268
|
+
"bt2020",
|
|
269
|
+
"-color_trc:v",
|
|
270
|
+
options.hdr.transfer === "pq" ? "smpte2084" : "arib-std-b67",
|
|
271
|
+
"-color_range",
|
|
272
|
+
"tv",
|
|
273
|
+
);
|
|
274
|
+
} else {
|
|
275
|
+
args.push(
|
|
276
|
+
"-colorspace:v",
|
|
277
|
+
"bt709",
|
|
278
|
+
"-color_primaries:v",
|
|
279
|
+
"bt709",
|
|
280
|
+
"-color_trc:v",
|
|
281
|
+
"bt709",
|
|
282
|
+
"-color_range",
|
|
283
|
+
"tv",
|
|
284
|
+
);
|
|
285
|
+
}
|
|
210
286
|
|
|
211
|
-
//
|
|
212
|
-
|
|
287
|
+
// Video filter for range/color conversion.
|
|
288
|
+
// Raw HDR input (from WebGPU pipeline) is already PQ-encoded — no conversion needed.
|
|
289
|
+
// Chrome screenshots need full→TV range conversion.
|
|
290
|
+
if (options.rawInputFormat) {
|
|
291
|
+
// No filter needed — PQ data goes straight to encoder
|
|
292
|
+
} else if (gpuEncoder === "vaapi") {
|
|
213
293
|
const vfIdx = args.indexOf("-vf");
|
|
214
294
|
if (vfIdx !== -1) {
|
|
215
295
|
args[vfIdx + 1] = `scale=in_range=pc:out_range=tv,${args[vfIdx + 1]}`;
|
|
216
296
|
}
|
|
217
297
|
} else if (!shouldUseGpu) {
|
|
298
|
+
// Range conversion: Chrome screenshots are full-range RGB.
|
|
218
299
|
args.push("-vf", "scale=in_range=pc:out_range=tv");
|
|
219
300
|
}
|
|
220
301
|
|
|
@@ -304,7 +385,15 @@ export async function spawnStreamingEncoder(
|
|
|
304
385
|
if (exitStatus !== "running" || !ffmpeg.stdin || ffmpeg.stdin.destroyed) {
|
|
305
386
|
return false;
|
|
306
387
|
}
|
|
307
|
-
|
|
388
|
+
// Copy the buffer before writing — Node streams hold a reference to the
|
|
389
|
+
// provided buffer and drain it asynchronously. The HDR path's compositor
|
|
390
|
+
// reuses pre-allocated transOutput/normalCanvas buffers across frames,
|
|
391
|
+
// so without this copy the pipe would read partially-overwritten data
|
|
392
|
+
// and flicker. The SDR path doesn't invoke writeFrame at all (it pipes
|
|
393
|
+
// PNG files via encodeFramesFromDir), so the memcpy here is HDR-only
|
|
394
|
+
// and justified by correctness.
|
|
395
|
+
const copy = Buffer.from(buffer);
|
|
396
|
+
return ffmpeg.stdin.write(copy);
|
|
308
397
|
},
|
|
309
398
|
|
|
310
399
|
close: async (): Promise<StreamingEncoderResult> => {
|
|
@@ -312,9 +401,10 @@ export async function spawnStreamingEncoder(
|
|
|
312
401
|
if (signal) signal.removeEventListener("abort", onAbort);
|
|
313
402
|
|
|
314
403
|
// Close stdin to signal end of input
|
|
315
|
-
|
|
404
|
+
const stdin = ffmpeg.stdin;
|
|
405
|
+
if (stdin && !stdin.destroyed) {
|
|
316
406
|
await new Promise<void>((resolve) => {
|
|
317
|
-
|
|
407
|
+
stdin.end(() => resolve());
|
|
318
408
|
});
|
|
319
409
|
}
|
|
320
410
|
|