@hyperframes/engine 0.5.5 → 0.6.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +3 -22
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -3
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/services/audioMixer.d.ts.map +1 -1
- package/dist/services/audioMixer.js +6 -4
- package/dist/services/audioMixer.js.map +1 -1
- package/dist/services/browserManager.d.ts +0 -35
- package/dist/services/browserManager.d.ts.map +1 -1
- package/dist/services/browserManager.js +1 -113
- package/dist/services/browserManager.js.map +1 -1
- package/dist/services/chunkEncoder.d.ts.map +1 -1
- package/dist/services/chunkEncoder.js +0 -31
- package/dist/services/chunkEncoder.js.map +1 -1
- package/dist/services/frameCapture.d.ts.map +1 -1
- package/dist/services/frameCapture.js +12 -26
- package/dist/services/frameCapture.js.map +1 -1
- package/dist/services/screenshotService.d.ts.map +1 -1
- package/dist/services/screenshotService.js +0 -7
- package/dist/services/screenshotService.js.map +1 -1
- package/dist/services/streamingEncoder.d.ts.map +1 -1
- package/dist/services/streamingEncoder.js +0 -20
- package/dist/services/streamingEncoder.js.map +1 -1
- package/dist/services/videoFrameExtractor.d.ts +0 -20
- package/dist/services/videoFrameExtractor.d.ts.map +1 -1
- package/dist/services/videoFrameExtractor.js +7 -95
- package/dist/services/videoFrameExtractor.js.map +1 -1
- package/dist/services/videoFrameInjector.d.ts +1 -40
- package/dist/services/videoFrameInjector.d.ts.map +1 -1
- package/dist/services/videoFrameInjector.js +9 -64
- package/dist/services/videoFrameInjector.js.map +1 -1
- package/dist/utils/alphaBlit.d.ts +1 -1
- package/dist/utils/alphaBlit.d.ts.map +1 -1
- package/dist/utils/alphaBlit.js +6 -15
- package/dist/utils/alphaBlit.js.map +1 -1
- package/dist/utils/ffprobe.d.ts.map +1 -1
- package/dist/utils/ffprobe.js +1 -17
- package/dist/utils/ffprobe.js.map +1 -1
- package/package.json +2 -2
- package/src/config.test.ts +0 -7
- package/src/config.ts +4 -31
- package/src/index.ts +0 -2
- package/src/services/audioMixer.ts +6 -4
- package/src/services/browserManager.test.ts +2 -83
- package/src/services/browserManager.ts +1 -130
- package/src/services/chunkEncoder.ts +0 -36
- package/src/services/frameCapture.ts +11 -26
- package/src/services/screenshotService.ts +0 -8
- package/src/services/streamingEncoder.ts +0 -25
- package/src/services/videoFrameExtractor.test.ts +1 -117
- package/src/services/videoFrameExtractor.ts +7 -100
- package/src/services/videoFrameInjector.ts +11 -89
- package/src/utils/alphaBlit.test.ts +0 -10
- package/src/utils/alphaBlit.ts +7 -15
- package/src/utils/ffprobe.test.ts +0 -40
- package/src/utils/ffprobe.ts +1 -16
- package/src/services/screenshotService.test.ts +0 -92
- package/src/services/videoFrameInjector.test.ts +0 -145
|
@@ -136,119 +136,6 @@ async function probeBeginFrameSupport(browser: Browser): Promise<boolean> {
|
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
-
/**
|
|
140
|
-
* Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`.
|
|
141
|
-
*
|
|
142
|
-
* Caching the Promise (rather than the resolved value) deduplicates concurrent
|
|
143
|
-
* callers — the parallel coordinator runs N workers via `Promise.all`, so a
|
|
144
|
-
* `--workers 4` render against a no-GPU host would otherwise fire 4
|
|
145
|
-
* simultaneous probe Chromes. The first call assigns the Promise and every
|
|
146
|
-
* other concurrent caller awaits the same one, paying the ~240 ms probe cost
|
|
147
|
-
* exactly once per process lifetime.
|
|
148
|
-
*
|
|
149
|
-
* Exported for tests; production callers go through `resolveBrowserGpuMode`.
|
|
150
|
-
*/
|
|
151
|
-
export let _autoBrowserGpuModeCache: Promise<"software" | "hardware"> | undefined;
|
|
152
|
-
|
|
153
|
-
/** Test-only: reset the cached probe result. */
|
|
154
|
-
export function _resetAutoBrowserGpuModeCacheForTests(): void {
|
|
155
|
-
_autoBrowserGpuModeCache = undefined;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
/**
|
|
159
|
-
* Resolve `browserGpuMode` to a concrete `"software" | "hardware"` answer.
|
|
160
|
-
*
|
|
161
|
-
* For `"software"` / `"hardware"` this is a pure pass-through. For `"auto"`
|
|
162
|
-
* it launches a tiny Chrome with the platform's hardware GPU args, runs a
|
|
163
|
-
* one-shot WebGL availability probe, and falls back to `"software"` if
|
|
164
|
-
* hardware-mode WebGL is unavailable. The Promise is cached for the process
|
|
165
|
-
* lifetime, so concurrent callers (parallel workers) share the same probe.
|
|
166
|
-
*
|
|
167
|
-
* Any failure (Chrome launch error, navigation timeout, missing canvas API,
|
|
168
|
-
* etc.) is treated as a `"software"` fallback. The render path with
|
|
169
|
-
* SwiftShader always works, so a misclassification toward software is the
|
|
170
|
-
* safe failure mode; misclassifying toward hardware would error on the real
|
|
171
|
-
* render.
|
|
172
|
-
*/
|
|
173
|
-
export function resolveBrowserGpuMode(
|
|
174
|
-
mode: EngineConfig["browserGpuMode"],
|
|
175
|
-
options: {
|
|
176
|
-
chromePath?: string;
|
|
177
|
-
browserTimeout?: number;
|
|
178
|
-
platform?: NodeJS.Platform;
|
|
179
|
-
} = {},
|
|
180
|
-
): Promise<"software" | "hardware"> {
|
|
181
|
-
if (mode !== "auto") return Promise.resolve(mode);
|
|
182
|
-
if (_autoBrowserGpuModeCache) return _autoBrowserGpuModeCache;
|
|
183
|
-
|
|
184
|
-
_autoBrowserGpuModeCache = (async () => {
|
|
185
|
-
const platform = options.platform ?? process.platform;
|
|
186
|
-
const browserTimeout = options.browserTimeout ?? DEFAULT_CONFIG.browserTimeout;
|
|
187
|
-
const executablePath = options.chromePath ?? resolveHeadlessShellPath({});
|
|
188
|
-
|
|
189
|
-
const probeArgs = [
|
|
190
|
-
"--no-sandbox",
|
|
191
|
-
"--disable-setuid-sandbox",
|
|
192
|
-
"--disable-dev-shm-usage",
|
|
193
|
-
"--enable-webgl",
|
|
194
|
-
"--ignore-gpu-blocklist",
|
|
195
|
-
...getBrowserGpuArgs("hardware", platform),
|
|
196
|
-
];
|
|
197
|
-
|
|
198
|
-
const ppt = await getPuppeteer().catch(() => null);
|
|
199
|
-
if (!ppt) {
|
|
200
|
-
logResolvedBrowserGpuMode("software", "puppeteer unavailable");
|
|
201
|
-
return "software" as const;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
let probeBrowser: Browser | undefined;
|
|
205
|
-
try {
|
|
206
|
-
probeBrowser = await ppt.launch({
|
|
207
|
-
headless: true,
|
|
208
|
-
args: probeArgs,
|
|
209
|
-
defaultViewport: { width: 64, height: 64 },
|
|
210
|
-
executablePath,
|
|
211
|
-
timeout: browserTimeout,
|
|
212
|
-
});
|
|
213
|
-
const page = await probeBrowser.newPage();
|
|
214
|
-
const hasWebGL = await page.evaluate(() => {
|
|
215
|
-
try {
|
|
216
|
-
const c = document.createElement("canvas");
|
|
217
|
-
const gl =
|
|
218
|
-
c.getContext("webgl") ||
|
|
219
|
-
(c.getContext("experimental-webgl") as RenderingContext | null);
|
|
220
|
-
return gl !== null;
|
|
221
|
-
} catch {
|
|
222
|
-
return false;
|
|
223
|
-
}
|
|
224
|
-
});
|
|
225
|
-
const resolved = hasWebGL ? ("hardware" as const) : ("software" as const);
|
|
226
|
-
logResolvedBrowserGpuMode(resolved, hasWebGL ? "WebGL probe succeeded" : "WebGL unavailable");
|
|
227
|
-
return resolved;
|
|
228
|
-
} catch (err) {
|
|
229
|
-
logResolvedBrowserGpuMode(
|
|
230
|
-
"software",
|
|
231
|
-
`probe failed (${err instanceof Error ? err.message : String(err)})`,
|
|
232
|
-
);
|
|
233
|
-
return "software" as const;
|
|
234
|
-
} finally {
|
|
235
|
-
await probeBrowser?.close().catch(() => {});
|
|
236
|
-
}
|
|
237
|
-
})();
|
|
238
|
-
|
|
239
|
-
return _autoBrowserGpuModeCache;
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/**
|
|
243
|
-
* Single observability surface for the auto-detect outcome. Logged exactly
|
|
244
|
-
* once per process (the probe runs once); without this line, a regression
|
|
245
|
-
* to "always software even with a GPU present" would be invisible in
|
|
246
|
-
* production. Goes to stderr to stay out of stdout pipelines.
|
|
247
|
-
*/
|
|
248
|
-
function logResolvedBrowserGpuMode(resolved: "hardware" | "software", reason: string): void {
|
|
249
|
-
console.error(`[hyperframes] browserGpuMode auto → ${resolved} (${reason})`);
|
|
250
|
-
}
|
|
251
|
-
|
|
252
139
|
export async function acquireBrowser(
|
|
253
140
|
chromeArgs: string[],
|
|
254
141
|
config?: Partial<
|
|
@@ -378,8 +265,6 @@ export interface BuildChromeArgsOptions {
|
|
|
378
265
|
platform?: NodeJS.Platform;
|
|
379
266
|
}
|
|
380
267
|
|
|
381
|
-
const CANVAS_DRAW_ELEMENT_FEATURE_FLAG = "--enable-features=CanvasDrawElement";
|
|
382
|
-
|
|
383
268
|
export function buildChromeArgs(
|
|
384
269
|
options: BuildChromeArgsOptions,
|
|
385
270
|
config?: Partial<Pick<EngineConfig, "browserGpuMode" | "disableGpu" | "chromePath">>,
|
|
@@ -397,7 +282,6 @@ export function buildChromeArgs(
|
|
|
397
282
|
"--no-sandbox",
|
|
398
283
|
"--disable-setuid-sandbox",
|
|
399
284
|
"--disable-dev-shm-usage",
|
|
400
|
-
CANVAS_DRAW_ELEMENT_FEATURE_FLAG,
|
|
401
285
|
"--enable-webgl",
|
|
402
286
|
"--ignore-gpu-blocklist",
|
|
403
287
|
...getBrowserGpuArgs(browserGpuMode, platform),
|
|
@@ -457,20 +341,7 @@ function getBrowserGpuArgs(
|
|
|
457
341
|
platform: NodeJS.Platform,
|
|
458
342
|
): string[] {
|
|
459
343
|
if (mode === "software") {
|
|
460
|
-
|
|
461
|
-
// path (--use-angle=swiftshader) keeps working but Chrome emits a
|
|
462
|
-
// deprecation warning unless --enable-unsafe-swiftshader is also set.
|
|
463
|
-
// Despite the name, this is exactly the behaviour Chrome had before;
|
|
464
|
-
// the flag exists to make CPU rasterisation an explicit opt-in rather
|
|
465
|
-
// than an implicit fallback for end users on the open web.
|
|
466
|
-
return ["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"];
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
if (mode === "auto") {
|
|
470
|
-
// Should not reach here — `resolveBrowserGpuMode` collapses "auto" to
|
|
471
|
-
// "software" or "hardware" before args are built. Be defensive: software
|
|
472
|
-
// is the always-safe fallback.
|
|
473
|
-
return ["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"];
|
|
344
|
+
return ["--use-gl=angle", "--use-angle=swiftshader"];
|
|
474
345
|
}
|
|
475
346
|
|
|
476
347
|
switch (platform) {
|
|
@@ -139,43 +139,12 @@ export function buildEncoderArgs(
|
|
|
139
139
|
else args.push("-global_quality", String(quality));
|
|
140
140
|
break;
|
|
141
141
|
}
|
|
142
|
-
|
|
143
|
-
// Same B-frame story as the SW branch below — nvenc emits B-frames
|
|
144
|
-
// by default (qsv via b_strategy, vaapi too), and the negative-DTS
|
|
145
|
-
// freeze hits the same downstream players. The unconditional
|
|
146
|
-
// `-avoid_negative_ts make_zero` near the bottom of this function
|
|
147
|
-
// covers the mux level, but we belt-and-suspenders the encoder too
|
|
148
|
-
// so even tools that consume the chunk file directly (without going
|
|
149
|
-
// through our mux step) play correctly. videotoolbox doesn't accept
|
|
150
|
-
// `-bf` so it's skipped — videotoolbox h264 also doesn't emit
|
|
151
|
-
// negative DTS in practice on macOS Sonoma+.
|
|
152
|
-
if (
|
|
153
|
-
codec === "h264" &&
|
|
154
|
-
(gpuEncoder === "nvenc" || gpuEncoder === "qsv" || gpuEncoder === "vaapi")
|
|
155
|
-
) {
|
|
156
|
-
args.push("-bf", "0");
|
|
157
|
-
if (gpuEncoder === "qsv") {
|
|
158
|
-
args.push("-b_strategy", "0");
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
142
|
} else {
|
|
162
143
|
const encoderName = codec === "h264" ? "libx264" : "libx265";
|
|
163
144
|
args.push("-c:v", encoderName, "-preset", preset);
|
|
164
145
|
if (bitrate) args.push("-b:v", bitrate);
|
|
165
146
|
else args.push("-crf", String(quality));
|
|
166
147
|
|
|
167
|
-
// Disable B-frames. Standard h264 with B-frames produces negative DTS
|
|
168
|
-
// at the start of the stream (the first B-frame's decode order is
|
|
169
|
-
// "before" the first I-frame's presentation time). VS Code's video
|
|
170
|
-
// preview, several browser <video> pipelines, and some HW decoders
|
|
171
|
-
// freeze on the first frame when DTS is negative, so audio plays alone.
|
|
172
|
-
// -bf 0 makes PTS == DTS at every frame, eliminating the issue at the
|
|
173
|
-
// source. Quality cost is ~5–10% larger files at the same CRF — a
|
|
174
|
-
// worthwhile trade for "the file plays everywhere".
|
|
175
|
-
if (codec === "h264") {
|
|
176
|
-
args.push("-bf", "0");
|
|
177
|
-
}
|
|
178
|
-
|
|
179
148
|
// Encoder-specific params: anti-banding + color space tagging.
|
|
180
149
|
// aq-mode=3 redistributes bits to dark flat areas (gradients).
|
|
181
150
|
// For HDR x265 paths we additionally embed BT.2020 + transfer + HDR static
|
|
@@ -270,8 +239,6 @@ export function buildEncoderArgs(
|
|
|
270
239
|
args.push("-pix_fmt", pixelFormat);
|
|
271
240
|
}
|
|
272
241
|
|
|
273
|
-
args.push("-avoid_negative_ts", "make_zero");
|
|
274
|
-
|
|
275
242
|
args.push("-y", outputPath);
|
|
276
243
|
return args;
|
|
277
244
|
}
|
|
@@ -543,9 +510,6 @@ export async function muxVideoWithAudio(
|
|
|
543
510
|
} else {
|
|
544
511
|
args.push("-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart");
|
|
545
512
|
}
|
|
546
|
-
// PTS bases can diverge during mux and reintroduce negative DTS. See
|
|
547
|
-
// buildEncoderArgs for the full reasoning on why that breaks playback.
|
|
548
|
-
args.push("-avoid_negative_ts", "make_zero");
|
|
549
513
|
args.push("-shortest", "-y", outputPath);
|
|
550
514
|
|
|
551
515
|
const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
|
@@ -18,7 +18,6 @@ import {
|
|
|
18
18
|
releaseBrowser,
|
|
19
19
|
forceReleaseBrowser,
|
|
20
20
|
buildChromeArgs,
|
|
21
|
-
resolveBrowserGpuMode,
|
|
22
21
|
resolveHeadlessShellPath,
|
|
23
22
|
type CaptureMode,
|
|
24
23
|
} from "./browserManager.js";
|
|
@@ -114,22 +113,11 @@ export async function createCaptureSession(
|
|
|
114
113
|
const headlessShell = resolveHeadlessShellPath(config);
|
|
115
114
|
const isLinux = process.platform === "linux";
|
|
116
115
|
const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot;
|
|
117
|
-
// BeginFrame's screenshot does not honor a viewport `deviceScaleFactor`
|
|
118
|
-
// (the captured surface is sized by the OS window in CSS pixels regardless
|
|
119
|
-
// of `Emulation.setDeviceMetricsOverride`'s DPR). When supersampling we
|
|
120
|
-
// need explicit clip+scale on `Page.captureScreenshot`, so fall back to
|
|
121
|
-
// the screenshot path for any DPR > 1.
|
|
122
|
-
const supersampling = (options.deviceScaleFactor ?? 1) > 1;
|
|
123
116
|
const preMode: CaptureMode =
|
|
124
|
-
headlessShell && isLinux && !forceScreenshot
|
|
125
|
-
const requestedGpuMode = config?.browserGpuMode ?? DEFAULT_CONFIG.browserGpuMode;
|
|
126
|
-
const resolvedGpuMode = await resolveBrowserGpuMode(requestedGpuMode, {
|
|
127
|
-
chromePath: headlessShell ?? undefined,
|
|
128
|
-
browserTimeout: config?.browserTimeout,
|
|
129
|
-
});
|
|
117
|
+
headlessShell && isLinux && !forceScreenshot ? "beginframe" : "screenshot";
|
|
130
118
|
const chromeArgs = buildChromeArgs(
|
|
131
119
|
{ width: options.width, height: options.height, captureMode: preMode },
|
|
132
|
-
|
|
120
|
+
config,
|
|
133
121
|
);
|
|
134
122
|
|
|
135
123
|
const { browser, captureMode } = await acquireBrowser(chromeArgs, config);
|
|
@@ -400,14 +388,8 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
|
|
400
388
|
|
|
401
389
|
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
|
402
390
|
|
|
403
|
-
// Wait for all video elements to have
|
|
404
|
-
//
|
|
405
|
-
// actually rasterized and ready to paint — at >= 1 (HAVE_METADATA) we
|
|
406
|
-
// only know the dimensions, and the first <video> screenshot can come
|
|
407
|
-
// back as a black/blank rectangle. This bites compositions with two
|
|
408
|
-
// <video> elements of different codecs (h264 mp4 + VP9 webm) where the
|
|
409
|
-
// faster decoder lets the readiness check pass while the slower one
|
|
410
|
-
// hasn't painted, producing a black "first frame" for the slower clip.
|
|
391
|
+
// Wait for all video elements to have loaded metadata (dimensions + duration)
|
|
392
|
+
// Without this, frame 0 captures videos at their 300x150 default size.
|
|
411
393
|
// skipReadinessVideoIds excludes natively-extracted videos (e.g. HDR HEVC
|
|
412
394
|
// sources) whose frames come from ffmpeg out-of-band. videoMetadataHints
|
|
413
395
|
// supply intrinsic dimensions for skipped videos whose layout depends on
|
|
@@ -415,12 +397,12 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
|
|
415
397
|
const skipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
|
|
416
398
|
const videosReady = await pollPageExpression(
|
|
417
399
|
page,
|
|
418
|
-
`(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >=
|
|
400
|
+
`(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
|
|
419
401
|
pageReadyTimeout,
|
|
420
402
|
);
|
|
421
403
|
if (!videosReady) {
|
|
422
404
|
throw new Error(
|
|
423
|
-
`[FrameCapture] video
|
|
405
|
+
`[FrameCapture] video metadata not ready after ${pageReadyTimeout}ms. Video elements must load metadata before capture starts.`,
|
|
424
406
|
);
|
|
425
407
|
}
|
|
426
408
|
|
|
@@ -502,13 +484,16 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
|
|
502
484
|
|
|
503
485
|
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
|
|
504
486
|
|
|
505
|
-
//
|
|
487
|
+
// Wait for all video elements to have loaded metadata (dimensions + duration).
|
|
488
|
+
// Without this, frame 0 captures videos at their 300x150 default size.
|
|
489
|
+
// See screenshot-mode comment above for why skipReadinessVideoIds and
|
|
490
|
+
// videoMetadataHints are paired.
|
|
506
491
|
const beginframeSkipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
|
|
507
492
|
const videoDeadline =
|
|
508
493
|
Date.now() + (session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout);
|
|
509
494
|
while (Date.now() < videoDeadline) {
|
|
510
495
|
const videosReady = await page.evaluate(
|
|
511
|
-
`(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >=
|
|
496
|
+
`(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
|
|
512
497
|
);
|
|
513
498
|
if (videosReady) break;
|
|
514
499
|
await new Promise((r) => setTimeout(r, 100));
|
|
@@ -129,20 +129,12 @@ export async function beginFrameCapture(
|
|
|
129
129
|
export async function pageScreenshotCapture(page: Page, options: CaptureOptions): Promise<Buffer> {
|
|
130
130
|
const client = await getCdpSession(page);
|
|
131
131
|
const isPng = options.format === "png";
|
|
132
|
-
const dpr = options.deviceScaleFactor ?? 1;
|
|
133
|
-
// When supersampling, pass an explicit clip with `scale` so Chrome emits a
|
|
134
|
-
// screenshot at device-pixel dimensions (`width × height × dpr`). Without
|
|
135
|
-
// this, `Page.captureScreenshot` returns at CSS dimensions regardless of
|
|
136
|
-
// the viewport's deviceScaleFactor.
|
|
137
|
-
const clip =
|
|
138
|
-
dpr > 1 ? { x: 0, y: 0, width: options.width, height: options.height, scale: dpr } : undefined;
|
|
139
132
|
const result = await client.send("Page.captureScreenshot", {
|
|
140
133
|
format: isPng ? "png" : "jpeg",
|
|
141
134
|
quality: isPng ? undefined : (options.quality ?? 80),
|
|
142
135
|
fromSurface: true,
|
|
143
136
|
captureBeyondViewport: false,
|
|
144
137
|
optimizeForSpeed: !isPng,
|
|
145
|
-
...(clip ? { clip } : {}),
|
|
146
138
|
});
|
|
147
139
|
return Buffer.from(result.data, "base64");
|
|
148
140
|
}
|
|
@@ -221,33 +221,12 @@ export function buildStreamingArgs(
|
|
|
221
221
|
else args.push("-global_quality", String(quality));
|
|
222
222
|
break;
|
|
223
223
|
}
|
|
224
|
-
|
|
225
|
-
// Mirror SW branch: GPU h264 paths emit B-frames by default (nvenc, qsv,
|
|
226
|
-
// vaapi) and produce the same negative-DTS freeze for downstream players.
|
|
227
|
-
// See chunkEncoder.buildEncoderArgs for the full explanation.
|
|
228
|
-
if (
|
|
229
|
-
codec === "h264" &&
|
|
230
|
-
(gpuEncoder === "nvenc" || gpuEncoder === "qsv" || gpuEncoder === "vaapi")
|
|
231
|
-
) {
|
|
232
|
-
args.push("-bf", "0");
|
|
233
|
-
if (gpuEncoder === "qsv") {
|
|
234
|
-
args.push("-b_strategy", "0");
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
224
|
} else {
|
|
238
225
|
const encoderName = codec === "h264" ? "libx264" : "libx265";
|
|
239
226
|
args.push("-c:v", encoderName, "-preset", preset);
|
|
240
227
|
if (bitrate) args.push("-b:v", bitrate);
|
|
241
228
|
else args.push("-crf", String(quality));
|
|
242
229
|
|
|
243
|
-
// Mirrors chunkEncoder: disable B-frames for h264 so PTS == DTS, no
|
|
244
|
-
// negative DTS at stream start. Without this, files freeze on the
|
|
245
|
-
// first frame in VS Code preview, several browsers, and some HW
|
|
246
|
-
// decoders. See chunkEncoder.buildEncoderArgs for the full reasoning.
|
|
247
|
-
if (codec === "h264") {
|
|
248
|
-
args.push("-bf", "0");
|
|
249
|
-
}
|
|
250
|
-
|
|
251
230
|
// Encoder-specific params: anti-banding + color space tagging.
|
|
252
231
|
// For HDR, getHdrEncoderColorParams also emits the SMPTE ST 2086
|
|
253
232
|
// mastering-display and CTA-861.3 MaxCLL/MaxFALL SEI messages —
|
|
@@ -334,10 +313,6 @@ export function buildStreamingArgs(
|
|
|
334
313
|
args.push("-pix_fmt", pixelFormat);
|
|
335
314
|
}
|
|
336
315
|
|
|
337
|
-
// Belt-and-suspenders against negative DTS at stream start. See chunkEncoder
|
|
338
|
-
// for the full explanation; same playback compatibility class.
|
|
339
|
-
args.push("-avoid_negative_ts", "make_zero");
|
|
340
|
-
|
|
341
316
|
args.push("-y", outputPath);
|
|
342
317
|
return args;
|
|
343
318
|
}
|
|
@@ -1,13 +1,5 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
2
|
-
import {
|
|
3
|
-
existsSync,
|
|
4
|
-
mkdirSync,
|
|
5
|
-
mkdtempSync,
|
|
6
|
-
readFileSync,
|
|
7
|
-
readdirSync,
|
|
8
|
-
rmSync,
|
|
9
|
-
writeFileSync,
|
|
10
|
-
} from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
11
3
|
import { createHash } from "node:crypto";
|
|
12
4
|
import { join } from "node:path";
|
|
13
5
|
import { tmpdir } from "node:os";
|
|
@@ -17,9 +9,6 @@ import {
|
|
|
17
9
|
parseImageElements,
|
|
18
10
|
extractAllVideoFrames,
|
|
19
11
|
createFrameLookupTable,
|
|
20
|
-
resolveProjectRelativeSrc,
|
|
21
|
-
codecMayHaveAlpha,
|
|
22
|
-
decoderForCodec,
|
|
23
12
|
type VideoElement,
|
|
24
13
|
type ExtractedFrames,
|
|
25
14
|
} from "./videoFrameExtractor.js";
|
|
@@ -34,111 +23,6 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
|
|
|
34
23
|
// synthesized VFR fixture.
|
|
35
24
|
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
|
|
36
25
|
|
|
37
|
-
// Codec-based alpha defaulting replaces tag-based detection (the
|
|
38
|
-
// alpha_mode/ALPHA_MODE case bug — see ffprobe.test.ts for the regression
|
|
39
|
-
// pin on that). The extractor uses these helpers for two decisions:
|
|
40
|
-
// 1. whether to force the alpha-aware decoder (libvpx-vp9 for VP9, libvpx
|
|
41
|
-
// for VP8)
|
|
42
|
-
// 2. whether to default the cached frame format to PNG (with alpha) vs JPG
|
|
43
|
-
// The "default to capable" trade is small file-size growth on opaque VP9
|
|
44
|
-
// content for correctness on alpha-having content even when the sidecar tag
|
|
45
|
-
// is missing or muxed with the wrong case.
|
|
46
|
-
describe("codec alpha capability", () => {
|
|
47
|
-
it("flags VP9, VP8, and ProRes as alpha-capable", () => {
|
|
48
|
-
expect(codecMayHaveAlpha("vp9")).toBe(true);
|
|
49
|
-
expect(codecMayHaveAlpha("VP9")).toBe(true);
|
|
50
|
-
expect(codecMayHaveAlpha("vp8")).toBe(true);
|
|
51
|
-
expect(codecMayHaveAlpha("prores")).toBe(true);
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
it("does not flag h264 / h265 / mpeg4 (no alpha in their bitstreams)", () => {
|
|
55
|
-
expect(codecMayHaveAlpha("h264")).toBe(false);
|
|
56
|
-
expect(codecMayHaveAlpha("h265")).toBe(false);
|
|
57
|
-
expect(codecMayHaveAlpha("hevc")).toBe(false);
|
|
58
|
-
expect(codecMayHaveAlpha("mpeg4")).toBe(false);
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
it("treats undefined / empty input as non-alpha", () => {
|
|
62
|
-
expect(codecMayHaveAlpha(undefined)).toBe(false);
|
|
63
|
-
expect(codecMayHaveAlpha("")).toBe(false);
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
it("returns the alpha-aware decoder name for VP9 and VP8", () => {
|
|
67
|
-
expect(decoderForCodec("vp9")).toBe("libvpx-vp9");
|
|
68
|
-
expect(decoderForCodec("VP9")).toBe("libvpx-vp9");
|
|
69
|
-
expect(decoderForCodec("vp8")).toBe("libvpx");
|
|
70
|
-
});
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
// Regression: a long-standing footgun where `<video src="../assets/foo">`
|
|
74
|
-
// inside a sub-composition silently dropped the video from extraction. The
|
|
75
|
-
// browser's URL resolver clamps `..` at the served origin's root (so the
|
|
76
|
-
// page renders fine in the studio), but `path.join(projectDir, "../assets/foo")`
|
|
77
|
-
// normalizes to <parentOfProjectDir>/assets/foo, which doesn't exist —
|
|
78
|
-
// extraction skipped, no frame injection, rendered output shows the video's
|
|
79
|
-
// first decoded frame for the whole clip duration. The resolver now mirrors
|
|
80
|
-
// browser semantics by clamping any traversal that escapes the project root.
|
|
81
|
-
describe("resolveProjectRelativeSrc — sub-composition path clamping", () => {
|
|
82
|
-
let tmp: string;
|
|
83
|
-
|
|
84
|
-
beforeAll(() => {
|
|
85
|
-
tmp = mkdtempSync(join(tmpdir(), "hf-resolver-"));
|
|
86
|
-
mkdirSync(join(tmp, "project", "assets"), { recursive: true });
|
|
87
|
-
writeFileSync(join(tmp, "project", "assets", "foo.mp4"), "");
|
|
88
|
-
});
|
|
89
|
-
afterAll(() => {
|
|
90
|
-
rmSync(tmp, { recursive: true, force: true });
|
|
91
|
-
});
|
|
92
|
-
|
|
93
|
-
it("returns the literal join when the file exists at projectDir/src", () => {
|
|
94
|
-
const projectDir = join(tmp, "project");
|
|
95
|
-
expect(resolveProjectRelativeSrc("assets/foo.mp4", projectDir)).toBe(
|
|
96
|
-
join(projectDir, "assets/foo.mp4"),
|
|
97
|
-
);
|
|
98
|
-
});
|
|
99
|
-
|
|
100
|
-
it("clamps a leading `../` so `../assets/foo.mp4` resolves to assets/foo.mp4", () => {
|
|
101
|
-
const projectDir = join(tmp, "project");
|
|
102
|
-
expect(resolveProjectRelativeSrc("../assets/foo.mp4", projectDir)).toBe(
|
|
103
|
-
join(projectDir, "assets/foo.mp4"),
|
|
104
|
-
);
|
|
105
|
-
});
|
|
106
|
-
|
|
107
|
-
it("clamps multiple leading `../../../` segments", () => {
|
|
108
|
-
const projectDir = join(tmp, "project");
|
|
109
|
-
expect(resolveProjectRelativeSrc("../../../assets/foo.mp4", projectDir)).toBe(
|
|
110
|
-
join(projectDir, "assets/foo.mp4"),
|
|
111
|
-
);
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
it("clamps mid-path traversal that escapes baseDir (not just leading `..`)", () => {
|
|
115
|
-
// `assets/../../foo.mp4` collapses past projectDir via path.join — this
|
|
116
|
-
// case used to silently escape; the resolver now strips embedded `..`
|
|
117
|
-
// segments and re-anchors at the project root.
|
|
118
|
-
const projectDir = join(tmp, "project");
|
|
119
|
-
expect(resolveProjectRelativeSrc("assets/../../assets/foo.mp4", projectDir)).toBe(
|
|
120
|
-
join(projectDir, "assets/foo.mp4"),
|
|
121
|
-
);
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
it("returns the (non-existent) base-dir path on miss so callers get a stable error message", () => {
|
|
125
|
-
const projectDir = join(tmp, "project");
|
|
126
|
-
expect(resolveProjectRelativeSrc("../assets/missing.mp4", projectDir)).toBe(
|
|
127
|
-
join(projectDir, "../assets/missing.mp4"),
|
|
128
|
-
);
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
it("prefers compiled-dir over base-dir when the file exists in both", () => {
|
|
132
|
-
const projectDir = join(tmp, "project");
|
|
133
|
-
const compiledDir = join(tmp, "compiled");
|
|
134
|
-
mkdirSync(join(compiledDir, "assets"), { recursive: true });
|
|
135
|
-
writeFileSync(join(compiledDir, "assets", "foo.mp4"), "");
|
|
136
|
-
expect(resolveProjectRelativeSrc("assets/foo.mp4", projectDir, compiledDir)).toBe(
|
|
137
|
-
join(compiledDir, "assets/foo.mp4"),
|
|
138
|
-
);
|
|
139
|
-
});
|
|
140
|
-
});
|
|
141
|
-
|
|
142
26
|
describe("parseVideoElements", () => {
|
|
143
27
|
it("parses videos without an id or data-start attribute", () => {
|
|
144
28
|
const videos = parseVideoElements('<video src="clip.mp4"></video>');
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { spawn } from "child_process";
|
|
9
9
|
import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
|
|
10
|
-
import { isAbsolute, join
|
|
10
|
+
import { isAbsolute, join } from "path";
|
|
11
11
|
import { parseHTML } from "linkedom";
|
|
12
12
|
import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
|
|
13
13
|
import {
|
|
@@ -230,17 +230,8 @@ export async function extractVideoFramesRange(
|
|
|
230
230
|
if (isHdr && isMacOS) {
|
|
231
231
|
args.push("-hwaccel", "videotoolbox");
|
|
232
232
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
// has at least three known failure modes: case-sensitivity across ffmpeg
|
|
236
|
-
// versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
|
|
237
|
-
// and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
|
|
238
|
-
// silently strips alpha during decode and the bug doesn't surface until
|
|
239
|
-
// the rendered video is missing layers. Codec-based default has no such
|
|
240
|
-
// ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
|
|
241
|
-
// normally when it isn't.
|
|
242
|
-
if (codecMayHaveAlpha(metadata.videoCodec)) {
|
|
243
|
-
args.push("-c:v", decoderForCodec(metadata.videoCodec));
|
|
233
|
+
if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
|
|
234
|
+
args.push("-c:v", "libvpx-vp9");
|
|
244
235
|
}
|
|
245
236
|
args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
|
|
246
237
|
|
|
@@ -407,31 +398,9 @@ function resolveSegmentDuration(
|
|
|
407
398
|
return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
|
|
408
399
|
}
|
|
409
400
|
|
|
410
|
-
/**
|
|
411
|
-
* Codecs whose bitstream is allowed to carry an alpha channel. Default the
|
|
412
|
-
* extraction path to PNG output for these regardless of `metadata.hasAlpha`
|
|
413
|
-
* so a missed sidecar tag doesn't silently strip transparency. Opaque content
|
|
414
|
-
* encoded in one of these codecs pays a small file-size cost on the cached
|
|
415
|
-
* frames but stays correct on the rare case where alpha IS present and the
|
|
416
|
-
* tag was missed.
|
|
417
|
-
*/
|
|
418
|
-
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
|
|
419
|
-
|
|
420
|
-
export function codecMayHaveAlpha(codec: string | undefined): boolean {
|
|
421
|
-
return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
export function decoderForCodec(codec: string | undefined): string {
|
|
425
|
-
const c = (codec ?? "").toLowerCase();
|
|
426
|
-
if (c === "vp9") return "libvpx-vp9";
|
|
427
|
-
if (c === "vp8") return "libvpx";
|
|
428
|
-
return c;
|
|
429
|
-
}
|
|
430
|
-
|
|
431
401
|
function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
|
|
432
402
|
if (requested) return requested;
|
|
433
|
-
|
|
434
|
-
return "jpg";
|
|
403
|
+
return metadata.hasAlpha ? "png" : "jpg";
|
|
435
404
|
}
|
|
436
405
|
|
|
437
406
|
/**
|
|
@@ -490,54 +459,6 @@ async function convertVfrToCfr(
|
|
|
490
459
|
}
|
|
491
460
|
}
|
|
492
461
|
|
|
493
|
-
/**
|
|
494
|
-
* Resolve a relative `<video src>` to a filesystem path the way the browser
|
|
495
|
-
* resolves it as a URL. Browsers clamp `..` segments at the served origin's
|
|
496
|
-
* root; `path.join(projectDir, "../assets/foo")` does not. So a sub-comp
|
|
497
|
-
* `<video src="../assets/foo">` loads in the page (browser clamps to
|
|
498
|
-
* `<projectDir>/assets/foo`) but the filesystem-side resolver lands at
|
|
499
|
-
* `<parentOfProjectDir>/assets/foo` — file missing, extraction skipped,
|
|
500
|
-
* the rendered output shows the video's first frame for the whole clip.
|
|
501
|
-
*
|
|
502
|
-
* The clamp covers two escape patterns: leading `..` (`../assets/foo`) AND
|
|
503
|
-
* mid-path escapes (`assets/../../foo`) that `path.join` collapses past the
|
|
504
|
-
* project root silently. Both fall back to a project-rooted candidate that
|
|
505
|
-
* strips traversal from the resolved path.
|
|
506
|
-
*
|
|
507
|
-
* Returns the first existing candidate, or the base-dir join on miss so
|
|
508
|
-
* the caller's `existsSync` check produces a stable error path.
|
|
509
|
-
*/
|
|
510
|
-
export function resolveProjectRelativeSrc(
|
|
511
|
-
src: string,
|
|
512
|
-
baseDir: string,
|
|
513
|
-
compiledDir?: string,
|
|
514
|
-
): string {
|
|
515
|
-
const fromCompiled = compiledDir ? join(compiledDir, src) : null;
|
|
516
|
-
const fromBase = join(baseDir, src);
|
|
517
|
-
const candidates: string[] = [];
|
|
518
|
-
if (fromCompiled) candidates.push(fromCompiled);
|
|
519
|
-
candidates.push(fromBase);
|
|
520
|
-
// If the joined result escapes the project root (either via leading `..`
|
|
521
|
-
// or mid-path traversal that path.join collapsed past baseDir), retry
|
|
522
|
-
// with the basename re-anchored at the project root. This mirrors the
|
|
523
|
-
// browser URL clamp without relying on a particular `..` shape.
|
|
524
|
-
const baseAbs = resolve(baseDir);
|
|
525
|
-
const fromBaseAbs = resolve(fromBase);
|
|
526
|
-
if (!fromBaseAbs.startsWith(baseAbs + sep) && fromBaseAbs !== baseAbs) {
|
|
527
|
-
// Normalize first (`assets/../../assets/foo.mp4` → `../assets/foo.mp4`)
|
|
528
|
-
// then strip any remaining leading `..` segments. Stripping `..` from the
|
|
529
|
-
// raw input would leave dangling siblings (`assets/../../assets/foo`
|
|
530
|
-
// would become `assets/assets/foo` instead of `assets/foo`).
|
|
531
|
-
const normalized = posix.normalize(src.replace(/\\/g, "/"));
|
|
532
|
-
const stripped = normalized.replace(/^(\.\.\/)+/, "");
|
|
533
|
-
if (stripped && stripped !== src && !stripped.startsWith("..")) {
|
|
534
|
-
if (compiledDir) candidates.push(join(compiledDir, stripped));
|
|
535
|
-
candidates.push(join(baseDir, stripped));
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
return candidates.find(existsSync) ?? fromBase;
|
|
539
|
-
}
|
|
540
|
-
|
|
541
462
|
export async function extractAllVideoFrames(
|
|
542
463
|
videos: VideoElement[],
|
|
543
464
|
baseDir: string,
|
|
@@ -566,9 +487,6 @@ export async function extractAllVideoFrames(
|
|
|
566
487
|
// Phase 1: Resolve paths and download remote videos
|
|
567
488
|
const phase1Start = Date.now();
|
|
568
489
|
const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
|
|
569
|
-
// Dedupe missing-src warnings: a composition with N <video> elements all
|
|
570
|
-
// pointing at the same broken src should only print one warning, not N.
|
|
571
|
-
const warnedSrcs = new Set<string>();
|
|
572
490
|
for (const video of videos) {
|
|
573
491
|
if (signal?.aborted) break;
|
|
574
492
|
try {
|
|
@@ -578,7 +496,9 @@ export async function extractAllVideoFrames(
|
|
|
578
496
|
// baseDir and produce duplicated, nonexistent paths
|
|
579
497
|
// (e.g. C:\tmp\hf-vfr-test-X\C:\tmp\hf-vfr-test-X\vfr_screen.mp4).
|
|
580
498
|
if (!isAbsolute(videoPath) && !isHttpUrl(videoPath)) {
|
|
581
|
-
|
|
499
|
+
const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
|
|
500
|
+
videoPath =
|
|
501
|
+
fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
|
|
582
502
|
}
|
|
583
503
|
|
|
584
504
|
if (isHttpUrl(videoPath)) {
|
|
@@ -588,19 +508,6 @@ export async function extractAllVideoFrames(
|
|
|
588
508
|
}
|
|
589
509
|
|
|
590
510
|
if (!existsSync(videoPath)) {
|
|
591
|
-
// Loud: silent miss leaves the rendered video frozen at frame 0 with
|
|
592
|
-
// no error in stdout — extremely confusing for authors. Dedupe by
|
|
593
|
-
// src so 50 broken videos pointing at the same path don't spam.
|
|
594
|
-
if (!warnedSrcs.has(video.src)) {
|
|
595
|
-
warnedSrcs.add(video.src);
|
|
596
|
-
process.stderr.write(
|
|
597
|
-
`[hyperframes:render] WARNING: video src="${video.src}" ` +
|
|
598
|
-
`could not be resolved on disk (looked for ${videoPath}). ` +
|
|
599
|
-
`The rendered output will show this video's first frame for the entire clip duration. ` +
|
|
600
|
-
`If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
|
|
601
|
-
`(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
|
|
602
|
-
);
|
|
603
|
-
}
|
|
604
511
|
errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
|
|
605
512
|
continue;
|
|
606
513
|
}
|