@hyperframes/engine 0.5.5 → 0.6.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/config.d.ts +3 -22
  2. package/dist/config.d.ts.map +1 -1
  3. package/dist/config.js +1 -3
  4. package/dist/config.js.map +1 -1
  5. package/dist/index.d.ts +2 -2
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +2 -2
  8. package/dist/index.js.map +1 -1
  9. package/dist/services/audioMixer.d.ts.map +1 -1
  10. package/dist/services/audioMixer.js +6 -4
  11. package/dist/services/audioMixer.js.map +1 -1
  12. package/dist/services/browserManager.d.ts +0 -35
  13. package/dist/services/browserManager.d.ts.map +1 -1
  14. package/dist/services/browserManager.js +1 -113
  15. package/dist/services/browserManager.js.map +1 -1
  16. package/dist/services/chunkEncoder.d.ts.map +1 -1
  17. package/dist/services/chunkEncoder.js +0 -31
  18. package/dist/services/chunkEncoder.js.map +1 -1
  19. package/dist/services/frameCapture.d.ts.map +1 -1
  20. package/dist/services/frameCapture.js +12 -26
  21. package/dist/services/frameCapture.js.map +1 -1
  22. package/dist/services/screenshotService.d.ts.map +1 -1
  23. package/dist/services/screenshotService.js +0 -7
  24. package/dist/services/screenshotService.js.map +1 -1
  25. package/dist/services/streamingEncoder.d.ts.map +1 -1
  26. package/dist/services/streamingEncoder.js +0 -20
  27. package/dist/services/streamingEncoder.js.map +1 -1
  28. package/dist/services/videoFrameExtractor.d.ts +0 -20
  29. package/dist/services/videoFrameExtractor.d.ts.map +1 -1
  30. package/dist/services/videoFrameExtractor.js +7 -95
  31. package/dist/services/videoFrameExtractor.js.map +1 -1
  32. package/dist/services/videoFrameInjector.d.ts +1 -40
  33. package/dist/services/videoFrameInjector.d.ts.map +1 -1
  34. package/dist/services/videoFrameInjector.js +9 -64
  35. package/dist/services/videoFrameInjector.js.map +1 -1
  36. package/dist/utils/alphaBlit.d.ts +1 -1
  37. package/dist/utils/alphaBlit.d.ts.map +1 -1
  38. package/dist/utils/alphaBlit.js +6 -15
  39. package/dist/utils/alphaBlit.js.map +1 -1
  40. package/dist/utils/ffprobe.d.ts.map +1 -1
  41. package/dist/utils/ffprobe.js +1 -17
  42. package/dist/utils/ffprobe.js.map +1 -1
  43. package/package.json +2 -2
  44. package/src/config.test.ts +0 -7
  45. package/src/config.ts +4 -31
  46. package/src/index.ts +0 -2
  47. package/src/services/audioMixer.ts +6 -4
  48. package/src/services/browserManager.test.ts +2 -83
  49. package/src/services/browserManager.ts +1 -130
  50. package/src/services/chunkEncoder.ts +0 -36
  51. package/src/services/frameCapture.ts +11 -26
  52. package/src/services/screenshotService.ts +0 -8
  53. package/src/services/streamingEncoder.ts +0 -25
  54. package/src/services/videoFrameExtractor.test.ts +1 -117
  55. package/src/services/videoFrameExtractor.ts +7 -100
  56. package/src/services/videoFrameInjector.ts +11 -89
  57. package/src/utils/alphaBlit.test.ts +0 -10
  58. package/src/utils/alphaBlit.ts +7 -15
  59. package/src/utils/ffprobe.test.ts +0 -40
  60. package/src/utils/ffprobe.ts +1 -16
  61. package/src/services/screenshotService.test.ts +0 -92
  62. package/src/services/videoFrameInjector.test.ts +0 -145
@@ -136,119 +136,6 @@ async function probeBeginFrameSupport(browser: Browser): Promise<boolean> {
136
136
  }
137
137
  }
138
138
 
139
- /**
140
- * Cached *in-flight or resolved* probe Promise for `resolveBrowserGpuMode("auto", ...)`.
141
- *
142
- * Caching the Promise (rather than the resolved value) deduplicates concurrent
143
- * callers — the parallel coordinator runs N workers via `Promise.all`, so a
144
- * `--workers 4` render against a no-GPU host would otherwise fire 4
145
- * simultaneous probe Chromes. The first call assigns the Promise and every
146
- * other concurrent caller awaits the same one, paying the ~240 ms probe cost
147
- * exactly once per process lifetime.
148
- *
149
- * Exported for tests; production callers go through `resolveBrowserGpuMode`.
150
- */
151
- export let _autoBrowserGpuModeCache: Promise<"software" | "hardware"> | undefined;
152
-
153
- /** Test-only: reset the cached probe result. */
154
- export function _resetAutoBrowserGpuModeCacheForTests(): void {
155
- _autoBrowserGpuModeCache = undefined;
156
- }
157
-
158
- /**
159
- * Resolve `browserGpuMode` to a concrete `"software" | "hardware"` answer.
160
- *
161
- * For `"software"` / `"hardware"` this is a pure pass-through. For `"auto"`
162
- * it launches a tiny Chrome with the platform's hardware GPU args, runs a
163
- * one-shot WebGL availability probe, and falls back to `"software"` if
164
- * hardware-mode WebGL is unavailable. The Promise is cached for the process
165
- * lifetime, so concurrent callers (parallel workers) share the same probe.
166
- *
167
- * Any failure (Chrome launch error, navigation timeout, missing canvas API,
168
- * etc.) is treated as a `"software"` fallback. The render path with
169
- * SwiftShader always works, so a misclassification toward software is the
170
- * safe failure mode; misclassifying toward hardware would error on the real
171
- * render.
172
- */
173
- export function resolveBrowserGpuMode(
174
- mode: EngineConfig["browserGpuMode"],
175
- options: {
176
- chromePath?: string;
177
- browserTimeout?: number;
178
- platform?: NodeJS.Platform;
179
- } = {},
180
- ): Promise<"software" | "hardware"> {
181
- if (mode !== "auto") return Promise.resolve(mode);
182
- if (_autoBrowserGpuModeCache) return _autoBrowserGpuModeCache;
183
-
184
- _autoBrowserGpuModeCache = (async () => {
185
- const platform = options.platform ?? process.platform;
186
- const browserTimeout = options.browserTimeout ?? DEFAULT_CONFIG.browserTimeout;
187
- const executablePath = options.chromePath ?? resolveHeadlessShellPath({});
188
-
189
- const probeArgs = [
190
- "--no-sandbox",
191
- "--disable-setuid-sandbox",
192
- "--disable-dev-shm-usage",
193
- "--enable-webgl",
194
- "--ignore-gpu-blocklist",
195
- ...getBrowserGpuArgs("hardware", platform),
196
- ];
197
-
198
- const ppt = await getPuppeteer().catch(() => null);
199
- if (!ppt) {
200
- logResolvedBrowserGpuMode("software", "puppeteer unavailable");
201
- return "software" as const;
202
- }
203
-
204
- let probeBrowser: Browser | undefined;
205
- try {
206
- probeBrowser = await ppt.launch({
207
- headless: true,
208
- args: probeArgs,
209
- defaultViewport: { width: 64, height: 64 },
210
- executablePath,
211
- timeout: browserTimeout,
212
- });
213
- const page = await probeBrowser.newPage();
214
- const hasWebGL = await page.evaluate(() => {
215
- try {
216
- const c = document.createElement("canvas");
217
- const gl =
218
- c.getContext("webgl") ||
219
- (c.getContext("experimental-webgl") as RenderingContext | null);
220
- return gl !== null;
221
- } catch {
222
- return false;
223
- }
224
- });
225
- const resolved = hasWebGL ? ("hardware" as const) : ("software" as const);
226
- logResolvedBrowserGpuMode(resolved, hasWebGL ? "WebGL probe succeeded" : "WebGL unavailable");
227
- return resolved;
228
- } catch (err) {
229
- logResolvedBrowserGpuMode(
230
- "software",
231
- `probe failed (${err instanceof Error ? err.message : String(err)})`,
232
- );
233
- return "software" as const;
234
- } finally {
235
- await probeBrowser?.close().catch(() => {});
236
- }
237
- })();
238
-
239
- return _autoBrowserGpuModeCache;
240
- }
241
-
242
- /**
243
- * Single observability surface for the auto-detect outcome. Logged exactly
244
- * once per process (the probe runs once); without this line, a regression
245
- * to "always software even with a GPU present" would be invisible in
246
- * production. Goes to stderr to stay out of stdout pipelines.
247
- */
248
- function logResolvedBrowserGpuMode(resolved: "hardware" | "software", reason: string): void {
249
- console.error(`[hyperframes] browserGpuMode auto → ${resolved} (${reason})`);
250
- }
251
-
252
139
  export async function acquireBrowser(
253
140
  chromeArgs: string[],
254
141
  config?: Partial<
@@ -378,8 +265,6 @@ export interface BuildChromeArgsOptions {
378
265
  platform?: NodeJS.Platform;
379
266
  }
380
267
 
381
- const CANVAS_DRAW_ELEMENT_FEATURE_FLAG = "--enable-features=CanvasDrawElement";
382
-
383
268
  export function buildChromeArgs(
384
269
  options: BuildChromeArgsOptions,
385
270
  config?: Partial<Pick<EngineConfig, "browserGpuMode" | "disableGpu" | "chromePath">>,
@@ -397,7 +282,6 @@ export function buildChromeArgs(
397
282
  "--no-sandbox",
398
283
  "--disable-setuid-sandbox",
399
284
  "--disable-dev-shm-usage",
400
- CANVAS_DRAW_ELEMENT_FEATURE_FLAG,
401
285
  "--enable-webgl",
402
286
  "--ignore-gpu-blocklist",
403
287
  ...getBrowserGpuArgs(browserGpuMode, platform),
@@ -457,20 +341,7 @@ function getBrowserGpuArgs(
457
341
  platform: NodeJS.Platform,
458
342
  ): string[] {
459
343
  if (mode === "software") {
460
- // Chrome 120+ deprecated implicit SwiftShader fallback; the explicit
461
- // path (--use-angle=swiftshader) keeps working but Chrome emits a
462
- // deprecation warning unless --enable-unsafe-swiftshader is also set.
463
- // Despite the name, this is exactly the behaviour Chrome had before;
464
- // the flag exists to make CPU rasterisation an explicit opt-in rather
465
- // than an implicit fallback for end users on the open web.
466
- return ["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"];
467
- }
468
-
469
- if (mode === "auto") {
470
- // Should not reach here — `resolveBrowserGpuMode` collapses "auto" to
471
- // "software" or "hardware" before args are built. Be defensive: software
472
- // is the always-safe fallback.
473
- return ["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"];
344
+ return ["--use-gl=angle", "--use-angle=swiftshader"];
474
345
  }
475
346
 
476
347
  switch (platform) {
@@ -139,43 +139,12 @@ export function buildEncoderArgs(
139
139
  else args.push("-global_quality", String(quality));
140
140
  break;
141
141
  }
142
-
143
- // Same B-frame story as the SW branch below — nvenc emits B-frames
144
- // by default (qsv via b_strategy, vaapi too), and the negative-DTS
145
- // freeze hits the same downstream players. The unconditional
146
- // `-avoid_negative_ts make_zero` near the bottom of this function
147
- // covers the mux level, but we belt-and-suspenders the encoder too
148
- // so even tools that consume the chunk file directly (without going
149
- // through our mux step) play correctly. videotoolbox doesn't accept
150
- // `-bf` so it's skipped — videotoolbox h264 also doesn't emit
151
- // negative DTS in practice on macOS Sonoma+.
152
- if (
153
- codec === "h264" &&
154
- (gpuEncoder === "nvenc" || gpuEncoder === "qsv" || gpuEncoder === "vaapi")
155
- ) {
156
- args.push("-bf", "0");
157
- if (gpuEncoder === "qsv") {
158
- args.push("-b_strategy", "0");
159
- }
160
- }
161
142
  } else {
162
143
  const encoderName = codec === "h264" ? "libx264" : "libx265";
163
144
  args.push("-c:v", encoderName, "-preset", preset);
164
145
  if (bitrate) args.push("-b:v", bitrate);
165
146
  else args.push("-crf", String(quality));
166
147
 
167
- // Disable B-frames. Standard h264 with B-frames produces negative DTS
168
- // at the start of the stream (the first B-frame's decode order is
169
- // "before" the first I-frame's presentation time). VS Code's video
170
- // preview, several browser <video> pipelines, and some HW decoders
171
- // freeze on the first frame when DTS is negative, so audio plays alone.
172
- // -bf 0 makes PTS == DTS at every frame, eliminating the issue at the
173
- // source. Quality cost is ~5–10% larger files at the same CRF — a
174
- // worthwhile trade for "the file plays everywhere".
175
- if (codec === "h264") {
176
- args.push("-bf", "0");
177
- }
178
-
179
148
  // Encoder-specific params: anti-banding + color space tagging.
180
149
  // aq-mode=3 redistributes bits to dark flat areas (gradients).
181
150
  // For HDR x265 paths we additionally embed BT.2020 + transfer + HDR static
@@ -270,8 +239,6 @@ export function buildEncoderArgs(
270
239
  args.push("-pix_fmt", pixelFormat);
271
240
  }
272
241
 
273
- args.push("-avoid_negative_ts", "make_zero");
274
-
275
242
  args.push("-y", outputPath);
276
243
  return args;
277
244
  }
@@ -543,9 +510,6 @@ export async function muxVideoWithAudio(
543
510
  } else {
544
511
  args.push("-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart");
545
512
  }
546
- // PTS bases can diverge during mux and reintroduce negative DTS. See
547
- // buildEncoderArgs for the full reasoning on why that breaks playback.
548
- args.push("-avoid_negative_ts", "make_zero");
549
513
  args.push("-shortest", "-y", outputPath);
550
514
 
551
515
  const processTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
@@ -18,7 +18,6 @@ import {
18
18
  releaseBrowser,
19
19
  forceReleaseBrowser,
20
20
  buildChromeArgs,
21
- resolveBrowserGpuMode,
22
21
  resolveHeadlessShellPath,
23
22
  type CaptureMode,
24
23
  } from "./browserManager.js";
@@ -114,22 +113,11 @@ export async function createCaptureSession(
114
113
  const headlessShell = resolveHeadlessShellPath(config);
115
114
  const isLinux = process.platform === "linux";
116
115
  const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot;
117
- // BeginFrame's screenshot does not honor a viewport `deviceScaleFactor`
118
- // (the captured surface is sized by the OS window in CSS pixels regardless
119
- // of `Emulation.setDeviceMetricsOverride`'s DPR). When supersampling we
120
- // need explicit clip+scale on `Page.captureScreenshot`, so fall back to
121
- // the screenshot path for any DPR > 1.
122
- const supersampling = (options.deviceScaleFactor ?? 1) > 1;
123
116
  const preMode: CaptureMode =
124
- headlessShell && isLinux && !forceScreenshot && !supersampling ? "beginframe" : "screenshot";
125
- const requestedGpuMode = config?.browserGpuMode ?? DEFAULT_CONFIG.browserGpuMode;
126
- const resolvedGpuMode = await resolveBrowserGpuMode(requestedGpuMode, {
127
- chromePath: headlessShell ?? undefined,
128
- browserTimeout: config?.browserTimeout,
129
- });
117
+ headlessShell && isLinux && !forceScreenshot ? "beginframe" : "screenshot";
130
118
  const chromeArgs = buildChromeArgs(
131
119
  { width: options.width, height: options.height, captureMode: preMode },
132
- { ...config, browserGpuMode: resolvedGpuMode },
120
+ config,
133
121
  );
134
122
 
135
123
  const { browser, captureMode } = await acquireBrowser(chromeArgs, config);
@@ -400,14 +388,8 @@ export async function initializeSession(session: CaptureSession): Promise<void>
400
388
 
401
389
  await applyVideoMetadataHints(page, session.options.videoMetadataHints);
402
390
 
403
- // Wait for all video elements to have decoded their CURRENT frame, not
404
- // just metadata. readyState >= 2 (HAVE_CURRENT_DATA) means a frame is
405
- // actually rasterized and ready to paint — at >= 1 (HAVE_METADATA) we
406
- // only know the dimensions, and the first <video> screenshot can come
407
- // back as a black/blank rectangle. This bites compositions with two
408
- // <video> elements of different codecs (h264 mp4 + VP9 webm) where the
409
- // faster decoder lets the readiness check pass while the slower one
410
- // hasn't painted, producing a black "first frame" for the slower clip.
391
+ // Wait for all video elements to have loaded metadata (dimensions + duration)
392
+ // Without this, frame 0 captures videos at their 300x150 default size.
411
393
  // skipReadinessVideoIds excludes natively-extracted videos (e.g. HDR HEVC
412
394
  // sources) whose frames come from ffmpeg out-of-band. videoMetadataHints
413
395
  // supply intrinsic dimensions for skipped videos whose layout depends on
@@ -415,12 +397,12 @@ export async function initializeSession(session: CaptureSession): Promise<void>
415
397
  const skipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
416
398
  const videosReady = await pollPageExpression(
417
399
  page,
418
- `(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 2); })()`,
400
+ `(() => { const skip = new Set(${skipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
419
401
  pageReadyTimeout,
420
402
  );
421
403
  if (!videosReady) {
422
404
  throw new Error(
423
- `[FrameCapture] video first frame not decoded after ${pageReadyTimeout}ms. Video elements must reach readyState >= 2 (HAVE_CURRENT_DATA) before capture starts.`,
405
+ `[FrameCapture] video metadata not ready after ${pageReadyTimeout}ms. Video elements must load metadata before capture starts.`,
424
406
  );
425
407
  }
426
408
 
@@ -502,13 +484,16 @@ export async function initializeSession(session: CaptureSession): Promise<void>
502
484
 
503
485
  await applyVideoMetadataHints(page, session.options.videoMetadataHints);
504
486
 
505
- // Same readyState contract as the screenshot path above (>= 2 / HAVE_CURRENT_DATA).
487
+ // Wait for all video elements to have loaded metadata (dimensions + duration).
488
+ // Without this, frame 0 captures videos at their 300x150 default size.
489
+ // See screenshot-mode comment above for why skipReadinessVideoIds and
490
+ // videoMetadataHints are paired.
506
491
  const beginframeSkipIdsLiteral = JSON.stringify(session.options.skipReadinessVideoIds ?? []);
507
492
  const videoDeadline =
508
493
  Date.now() + (session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout);
509
494
  while (Date.now() < videoDeadline) {
510
495
  const videosReady = await page.evaluate(
511
- `(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 2); })()`,
496
+ `(() => { const skip = new Set(${beginframeSkipIdsLiteral}); const vids = Array.from(document.querySelectorAll("video")).filter(v => !skip.has(v.id)); return vids.length === 0 || vids.every(v => v.readyState >= 1); })()`,
512
497
  );
513
498
  if (videosReady) break;
514
499
  await new Promise((r) => setTimeout(r, 100));
@@ -129,20 +129,12 @@ export async function beginFrameCapture(
129
129
  export async function pageScreenshotCapture(page: Page, options: CaptureOptions): Promise<Buffer> {
130
130
  const client = await getCdpSession(page);
131
131
  const isPng = options.format === "png";
132
- const dpr = options.deviceScaleFactor ?? 1;
133
- // When supersampling, pass an explicit clip with `scale` so Chrome emits a
134
- // screenshot at device-pixel dimensions (`width × height × dpr`). Without
135
- // this, `Page.captureScreenshot` returns at CSS dimensions regardless of
136
- // the viewport's deviceScaleFactor.
137
- const clip =
138
- dpr > 1 ? { x: 0, y: 0, width: options.width, height: options.height, scale: dpr } : undefined;
139
132
  const result = await client.send("Page.captureScreenshot", {
140
133
  format: isPng ? "png" : "jpeg",
141
134
  quality: isPng ? undefined : (options.quality ?? 80),
142
135
  fromSurface: true,
143
136
  captureBeyondViewport: false,
144
137
  optimizeForSpeed: !isPng,
145
- ...(clip ? { clip } : {}),
146
138
  });
147
139
  return Buffer.from(result.data, "base64");
148
140
  }
@@ -221,33 +221,12 @@ export function buildStreamingArgs(
221
221
  else args.push("-global_quality", String(quality));
222
222
  break;
223
223
  }
224
-
225
- // Mirror SW branch: GPU h264 paths emit B-frames by default (nvenc, qsv,
226
- // vaapi) and produce the same negative-DTS freeze for downstream players.
227
- // See chunkEncoder.buildEncoderArgs for the full explanation.
228
- if (
229
- codec === "h264" &&
230
- (gpuEncoder === "nvenc" || gpuEncoder === "qsv" || gpuEncoder === "vaapi")
231
- ) {
232
- args.push("-bf", "0");
233
- if (gpuEncoder === "qsv") {
234
- args.push("-b_strategy", "0");
235
- }
236
- }
237
224
  } else {
238
225
  const encoderName = codec === "h264" ? "libx264" : "libx265";
239
226
  args.push("-c:v", encoderName, "-preset", preset);
240
227
  if (bitrate) args.push("-b:v", bitrate);
241
228
  else args.push("-crf", String(quality));
242
229
 
243
- // Mirrors chunkEncoder: disable B-frames for h264 so PTS == DTS, no
244
- // negative DTS at stream start. Without this, files freeze on the
245
- // first frame in VS Code preview, several browsers, and some HW
246
- // decoders. See chunkEncoder.buildEncoderArgs for the full reasoning.
247
- if (codec === "h264") {
248
- args.push("-bf", "0");
249
- }
250
-
251
230
  // Encoder-specific params: anti-banding + color space tagging.
252
231
  // For HDR, getHdrEncoderColorParams also emits the SMPTE ST 2086
253
232
  // mastering-display and CTA-861.3 MaxCLL/MaxFALL SEI messages —
@@ -334,10 +313,6 @@ export function buildStreamingArgs(
334
313
  args.push("-pix_fmt", pixelFormat);
335
314
  }
336
315
 
337
- // Belt-and-suspenders against negative DTS at stream start. See chunkEncoder
338
- // for the full explanation; same playback compatibility class.
339
- args.push("-avoid_negative_ts", "make_zero");
340
-
341
316
  args.push("-y", outputPath);
342
317
  return args;
343
318
  }
@@ -1,13 +1,5 @@
1
1
  import { afterAll, beforeAll, describe, expect, it } from "vitest";
2
- import {
3
- existsSync,
4
- mkdirSync,
5
- mkdtempSync,
6
- readFileSync,
7
- readdirSync,
8
- rmSync,
9
- writeFileSync,
10
- } from "node:fs";
2
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
11
3
  import { createHash } from "node:crypto";
12
4
  import { join } from "node:path";
13
5
  import { tmpdir } from "node:os";
@@ -17,9 +9,6 @@ import {
17
9
  parseImageElements,
18
10
  extractAllVideoFrames,
19
11
  createFrameLookupTable,
20
- resolveProjectRelativeSrc,
21
- codecMayHaveAlpha,
22
- decoderForCodec,
23
12
  type VideoElement,
24
13
  type ExtractedFrames,
25
14
  } from "./videoFrameExtractor.js";
@@ -34,111 +23,6 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
34
23
  // synthesized VFR fixture.
35
24
  const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
36
25
 
37
- // Codec-based alpha defaulting replaces tag-based detection (the
38
- // alpha_mode/ALPHA_MODE case bug — see ffprobe.test.ts for the regression
39
- // pin on that). The extractor uses these helpers for two decisions:
40
- // 1. whether to force the alpha-aware decoder (libvpx-vp9 for VP9, libvpx
41
- // for VP8)
42
- // 2. whether to default the cached frame format to PNG (with alpha) vs JPG
43
- // The "default to capable" trade is small file-size growth on opaque VP9
44
- // content for correctness on alpha-having content even when the sidecar tag
45
- // is missing or muxed with the wrong case.
46
- describe("codec alpha capability", () => {
47
- it("flags VP9, VP8, and ProRes as alpha-capable", () => {
48
- expect(codecMayHaveAlpha("vp9")).toBe(true);
49
- expect(codecMayHaveAlpha("VP9")).toBe(true);
50
- expect(codecMayHaveAlpha("vp8")).toBe(true);
51
- expect(codecMayHaveAlpha("prores")).toBe(true);
52
- });
53
-
54
- it("does not flag h264 / h265 / mpeg4 (no alpha in their bitstreams)", () => {
55
- expect(codecMayHaveAlpha("h264")).toBe(false);
56
- expect(codecMayHaveAlpha("h265")).toBe(false);
57
- expect(codecMayHaveAlpha("hevc")).toBe(false);
58
- expect(codecMayHaveAlpha("mpeg4")).toBe(false);
59
- });
60
-
61
- it("treats undefined / empty input as non-alpha", () => {
62
- expect(codecMayHaveAlpha(undefined)).toBe(false);
63
- expect(codecMayHaveAlpha("")).toBe(false);
64
- });
65
-
66
- it("returns the alpha-aware decoder name for VP9 and VP8", () => {
67
- expect(decoderForCodec("vp9")).toBe("libvpx-vp9");
68
- expect(decoderForCodec("VP9")).toBe("libvpx-vp9");
69
- expect(decoderForCodec("vp8")).toBe("libvpx");
70
- });
71
- });
72
-
73
- // Regression: a long-standing footgun where `<video src="../assets/foo">`
74
- // inside a sub-composition silently dropped the video from extraction. The
75
- // browser's URL resolver clamps `..` at the served origin's root (so the
76
- // page renders fine in the studio), but `path.join(projectDir, "../assets/foo")`
77
- // normalizes to <parentOfProjectDir>/assets/foo, which doesn't exist —
78
- // extraction skipped, no frame injection, rendered output shows the video's
79
- // first decoded frame for the whole clip duration. The resolver now mirrors
80
- // browser semantics by clamping any traversal that escapes the project root.
81
- describe("resolveProjectRelativeSrc — sub-composition path clamping", () => {
82
- let tmp: string;
83
-
84
- beforeAll(() => {
85
- tmp = mkdtempSync(join(tmpdir(), "hf-resolver-"));
86
- mkdirSync(join(tmp, "project", "assets"), { recursive: true });
87
- writeFileSync(join(tmp, "project", "assets", "foo.mp4"), "");
88
- });
89
- afterAll(() => {
90
- rmSync(tmp, { recursive: true, force: true });
91
- });
92
-
93
- it("returns the literal join when the file exists at projectDir/src", () => {
94
- const projectDir = join(tmp, "project");
95
- expect(resolveProjectRelativeSrc("assets/foo.mp4", projectDir)).toBe(
96
- join(projectDir, "assets/foo.mp4"),
97
- );
98
- });
99
-
100
- it("clamps a leading `../` so `../assets/foo.mp4` resolves to assets/foo.mp4", () => {
101
- const projectDir = join(tmp, "project");
102
- expect(resolveProjectRelativeSrc("../assets/foo.mp4", projectDir)).toBe(
103
- join(projectDir, "assets/foo.mp4"),
104
- );
105
- });
106
-
107
- it("clamps multiple leading `../../../` segments", () => {
108
- const projectDir = join(tmp, "project");
109
- expect(resolveProjectRelativeSrc("../../../assets/foo.mp4", projectDir)).toBe(
110
- join(projectDir, "assets/foo.mp4"),
111
- );
112
- });
113
-
114
- it("clamps mid-path traversal that escapes baseDir (not just leading `..`)", () => {
115
- // `assets/../../foo.mp4` collapses past projectDir via path.join — this
116
- // case used to silently escape; the resolver now strips embedded `..`
117
- // segments and re-anchors at the project root.
118
- const projectDir = join(tmp, "project");
119
- expect(resolveProjectRelativeSrc("assets/../../assets/foo.mp4", projectDir)).toBe(
120
- join(projectDir, "assets/foo.mp4"),
121
- );
122
- });
123
-
124
- it("returns the (non-existent) base-dir path on miss so callers get a stable error message", () => {
125
- const projectDir = join(tmp, "project");
126
- expect(resolveProjectRelativeSrc("../assets/missing.mp4", projectDir)).toBe(
127
- join(projectDir, "../assets/missing.mp4"),
128
- );
129
- });
130
-
131
- it("prefers compiled-dir over base-dir when the file exists in both", () => {
132
- const projectDir = join(tmp, "project");
133
- const compiledDir = join(tmp, "compiled");
134
- mkdirSync(join(compiledDir, "assets"), { recursive: true });
135
- writeFileSync(join(compiledDir, "assets", "foo.mp4"), "");
136
- expect(resolveProjectRelativeSrc("assets/foo.mp4", projectDir, compiledDir)).toBe(
137
- join(compiledDir, "assets/foo.mp4"),
138
- );
139
- });
140
- });
141
-
142
26
  describe("parseVideoElements", () => {
143
27
  it("parses videos without an id or data-start attribute", () => {
144
28
  const videos = parseVideoElements('<video src="clip.mp4"></video>');
@@ -7,7 +7,7 @@
7
7
 
8
8
  import { spawn } from "child_process";
9
9
  import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
10
- import { isAbsolute, join, posix, resolve, sep } from "path";
10
+ import { isAbsolute, join } from "path";
11
11
  import { parseHTML } from "linkedom";
12
12
  import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
13
13
  import {
@@ -230,17 +230,8 @@ export async function extractVideoFramesRange(
230
230
  if (isHdr && isMacOS) {
231
231
  args.push("-hwaccel", "videotoolbox");
232
232
  }
233
- // Always force the alpha-aware decoder on codecs that can carry alpha. The
234
- // alternative — gating on `metadata.hasAlpha` — relies on tag detection that
235
- // has at least three known failure modes: case-sensitivity across ffmpeg
236
- // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
237
- // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
238
- // silently strips alpha during decode and the bug doesn't surface until
239
- // the rendered video is missing layers. Codec-based default has no such
240
- // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
241
- // normally when it isn't.
242
- if (codecMayHaveAlpha(metadata.videoCodec)) {
243
- args.push("-c:v", decoderForCodec(metadata.videoCodec));
233
+ if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
234
+ args.push("-c:v", "libvpx-vp9");
244
235
  }
245
236
  args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
246
237
 
@@ -407,31 +398,9 @@ function resolveSegmentDuration(
407
398
  return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
408
399
  }
409
400
 
410
- /**
411
- * Codecs whose bitstream is allowed to carry an alpha channel. Default the
412
- * extraction path to PNG output for these regardless of `metadata.hasAlpha`
413
- * so a missed sidecar tag doesn't silently strip transparency. Opaque content
414
- * encoded in one of these codecs pays a small file-size cost on the cached
415
- * frames but stays correct on the rare case where alpha IS present and the
416
- * tag was missed.
417
- */
418
- const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
419
-
420
- export function codecMayHaveAlpha(codec: string | undefined): boolean {
421
- return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
422
- }
423
-
424
- export function decoderForCodec(codec: string | undefined): string {
425
- const c = (codec ?? "").toLowerCase();
426
- if (c === "vp9") return "libvpx-vp9";
427
- if (c === "vp8") return "libvpx";
428
- return c;
429
- }
430
-
431
401
  function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
432
402
  if (requested) return requested;
433
- if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
434
- return "jpg";
403
+ return metadata.hasAlpha ? "png" : "jpg";
435
404
  }
436
405
 
437
406
  /**
@@ -490,54 +459,6 @@ async function convertVfrToCfr(
490
459
  }
491
460
  }
492
461
 
493
- /**
494
- * Resolve a relative `<video src>` to a filesystem path the way the browser
495
- * resolves it as a URL. Browsers clamp `..` segments at the served origin's
496
- * root; `path.join(projectDir, "../assets/foo")` does not. So a sub-comp
497
- * `<video src="../assets/foo">` loads in the page (browser clamps to
498
- * `<projectDir>/assets/foo`) but the filesystem-side resolver lands at
499
- * `<parentOfProjectDir>/assets/foo` — file missing, extraction skipped,
500
- * the rendered output shows the video's first frame for the whole clip.
501
- *
502
- * The clamp covers two escape patterns: leading `..` (`../assets/foo`) AND
503
- * mid-path escapes (`assets/../../foo`) that `path.join` collapses past the
504
- * project root silently. Both fall back to a project-rooted candidate that
505
- * strips traversal from the resolved path.
506
- *
507
- * Returns the first existing candidate, or the base-dir join on miss so
508
- * the caller's `existsSync` check produces a stable error path.
509
- */
510
- export function resolveProjectRelativeSrc(
511
- src: string,
512
- baseDir: string,
513
- compiledDir?: string,
514
- ): string {
515
- const fromCompiled = compiledDir ? join(compiledDir, src) : null;
516
- const fromBase = join(baseDir, src);
517
- const candidates: string[] = [];
518
- if (fromCompiled) candidates.push(fromCompiled);
519
- candidates.push(fromBase);
520
- // If the joined result escapes the project root (either via leading `..`
521
- // or mid-path traversal that path.join collapsed past baseDir), retry
522
- // with the basename re-anchored at the project root. This mirrors the
523
- // browser URL clamp without relying on a particular `..` shape.
524
- const baseAbs = resolve(baseDir);
525
- const fromBaseAbs = resolve(fromBase);
526
- if (!fromBaseAbs.startsWith(baseAbs + sep) && fromBaseAbs !== baseAbs) {
527
- // Normalize first (`assets/../../assets/foo.mp4` → `../assets/foo.mp4`)
528
- // then strip any remaining leading `..` segments. Stripping `..` from the
529
- // raw input would leave dangling siblings (`assets/../../assets/foo`
530
- // would become `assets/assets/foo` instead of `assets/foo`).
531
- const normalized = posix.normalize(src.replace(/\\/g, "/"));
532
- const stripped = normalized.replace(/^(\.\.\/)+/, "");
533
- if (stripped && stripped !== src && !stripped.startsWith("..")) {
534
- if (compiledDir) candidates.push(join(compiledDir, stripped));
535
- candidates.push(join(baseDir, stripped));
536
- }
537
- }
538
- return candidates.find(existsSync) ?? fromBase;
539
- }
540
-
541
462
  export async function extractAllVideoFrames(
542
463
  videos: VideoElement[],
543
464
  baseDir: string,
@@ -566,9 +487,6 @@ export async function extractAllVideoFrames(
566
487
  // Phase 1: Resolve paths and download remote videos
567
488
  const phase1Start = Date.now();
568
489
  const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
569
- // Dedupe missing-src warnings: a composition with N <video> elements all
570
- // pointing at the same broken src should only print one warning, not N.
571
- const warnedSrcs = new Set<string>();
572
490
  for (const video of videos) {
573
491
  if (signal?.aborted) break;
574
492
  try {
@@ -578,7 +496,9 @@ export async function extractAllVideoFrames(
578
496
  // baseDir and produce duplicated, nonexistent paths
579
497
  // (e.g. C:\tmp\hf-vfr-test-X\C:\tmp\hf-vfr-test-X\vfr_screen.mp4).
580
498
  if (!isAbsolute(videoPath) && !isHttpUrl(videoPath)) {
581
- videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
499
+ const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
500
+ videoPath =
501
+ fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
582
502
  }
583
503
 
584
504
  if (isHttpUrl(videoPath)) {
@@ -588,19 +508,6 @@ export async function extractAllVideoFrames(
588
508
  }
589
509
 
590
510
  if (!existsSync(videoPath)) {
591
- // Loud: silent miss leaves the rendered video frozen at frame 0 with
592
- // no error in stdout — extremely confusing for authors. Dedupe by
593
- // src so 50 broken videos pointing at the same path don't spam.
594
- if (!warnedSrcs.has(video.src)) {
595
- warnedSrcs.add(video.src);
596
- process.stderr.write(
597
- `[hyperframes:render] WARNING: video src="${video.src}" ` +
598
- `could not be resolved on disk (looked for ${videoPath}). ` +
599
- `The rendered output will show this video's first frame for the entire clip duration. ` +
600
- `If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
601
- `(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
602
- );
603
- }
604
511
  errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
605
512
  continue;
606
513
  }