@hyperframes/engine 0.4.5 → 0.4.7

package/src/services/videoFrameExtractor.ts

@@ -10,7 +10,9 @@ import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
 import { join } from "path";
 import { parseHTML } from "linkedom";
 import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js";
+import { isHdrColorSpace as isHdrColorSpaceUtil } from "../utils/hdr.js";
 import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
+import { runFfmpeg } from "../utils/runFfmpeg.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 
 export interface VideoElement {
@@ -114,18 +116,28 @@ export async function extractVideoFramesRange(
   const framePattern = `frame_%05d.${format}`;
   const outputPattern = join(videoOutputDir, framePattern);
 
-  const args: string[] = [
-    "-ss",
-    String(startTime),
-    "-i",
-    videoPath,
-    "-t",
-    String(duration),
-    "-vf",
-    `fps=${fps}`,
-    "-q:v",
-    format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0",
-  ];
+  // When extracting from HDR source, tone-map to SDR in FFmpeg rather than
+  // letting Chrome's uncontrollable tone-mapper handle it (which washes out).
+  // macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon.
+  // Linux: zscale filter (when available) or colorspace filter as fallback.
+  const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
+  const isMacOS = process.platform === "darwin";
+
+  const args: string[] = [];
+  if (isHdr && isMacOS) {
+    args.push("-hwaccel", "videotoolbox");
+  }
+  args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
+
+  const vfFilters: string[] = [];
+  if (isHdr && isMacOS) {
+    // VideoToolbox tone-maps during decode; force output to bt709 SDR format
+    vfFilters.push("format=nv12");
+  }
+  vfFilters.push(`fps=${fps}`);
+  args.push("-vf", vfFilters.join(","));
+
+  args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
   if (format === "png") args.push("-compression_level", "6");
   args.push("-y", outputPattern);
 
@@ -195,6 +207,53 @@ export async function extractVideoFramesRange(
   });
 }
 
+/**
+ * Convert an SDR video to HDR color space (HLG / BT.2020) so it can be
+ * composited alongside HDR content without looking washed out.
+ *
+ * Uses zscale for color space conversion with a nominal peak luminance of
+ * 600 nits — high enough that SDR content doesn't appear too dark next to
+ * HDR, matching the approach used by HeyGen's Rio pipeline.
+ */
+async function convertSdrToHdr(
+  inputPath: string,
+  outputPath: string,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<void> {
+  const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+
+  const args = [
+    "-i",
+    inputPath,
+    "-vf",
+    "colorspace=all=bt2020:iall=bt709:range=tv",
+    "-color_primaries",
+    "bt2020",
+    "-color_trc",
+    "arib-std-b67",
+    "-colorspace",
+    "bt2020nc",
+    "-c:v",
+    "libx264",
+    "-preset",
+    "fast",
+    "-crf",
+    "16",
+    "-c:a",
+    "copy",
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout });
+  if (!result.success) {
+    throw new Error(
+      `SDR→HDR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`,
+    );
+  }
+}
+
 export async function extractAllVideoFrames(
   videos: VideoElement[],
   baseDir: string,
@@ -208,30 +267,75 @@ export async function extractAllVideoFrames(
   const errors: Array<{ videoId: string; error: string }> = [];
   let totalFramesExtracted = 0;
 
-  // Process videos in parallel for better performance
-  const results = await Promise.all(
-    videos.map(async (video) => {
-      if (signal?.aborted) {
-        throw new Error("Video frame extraction cancelled");
+  // Phase 1: Resolve paths and download remote videos
+  const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
+  for (const video of videos) {
+    if (signal?.aborted) break;
+    try {
+      let videoPath = video.src;
+      if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
+        const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
+        videoPath =
+          fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
       }
-      try {
-        let videoPath = video.src;
-        if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
-          const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
-          videoPath =
-            fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
-        }
 
-        if (isHttpUrl(videoPath)) {
-          const downloadDir = join(options.outputDir, "_downloads");
-          mkdirSync(downloadDir, { recursive: true });
-          videoPath = await downloadToTemp(videoPath, downloadDir);
-        }
+      if (isHttpUrl(videoPath)) {
+        const downloadDir = join(options.outputDir, "_downloads");
+        mkdirSync(downloadDir, { recursive: true });
+        videoPath = await downloadToTemp(videoPath, downloadDir);
+      }
+
+      if (!existsSync(videoPath)) {
+        errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
+        continue;
+      }
+      resolvedVideos.push({ video, videoPath });
+    } catch (err) {
+      errors.push({ videoId: video.id, error: err instanceof Error ? err.message : String(err) });
+    }
+  }
 
-        if (!existsSync(videoPath)) {
-          return { error: { videoId: video.id, error: `Video file not found: ${videoPath}` } };
+  // Phase 2: Probe color spaces and normalize if mixed HDR/SDR
+  const videoColorSpaces = await Promise.all(
+    resolvedVideos.map(async ({ videoPath }) => {
+      const metadata = await extractVideoMetadata(videoPath);
+      return metadata.colorSpace;
+    }),
+  );
+
+  const hasAnyHdr = videoColorSpaces.some(isHdrColorSpaceUtil);
+  if (hasAnyHdr) {
+    const convertDir = join(options.outputDir, "_hdr_normalized");
+    mkdirSync(convertDir, { recursive: true });
+
+    for (let i = 0; i < resolvedVideos.length; i++) {
+      if (signal?.aborted) break;
+      const cs = videoColorSpaces[i] ?? null;
+      if (!isHdrColorSpaceUtil(cs)) {
+        // SDR video in a mixed timeline — convert to HDR color space
+        const entry = resolvedVideos[i];
+        if (!entry) continue;
+        const convertedPath = join(convertDir, `${entry.video.id}_hdr.mp4`);
+        try {
+          await convertSdrToHdr(entry.videoPath, convertedPath, signal, config);
+          entry.videoPath = convertedPath;
+        } catch (err) {
+          errors.push({
+            videoId: entry.video.id,
+            error: `SDR→HDR conversion failed: ${err instanceof Error ? err.message : String(err)}`,
+          });
         }
+      }
+    }
+  }
 
+  // Phase 3: Extract frames (parallel)
+  const results = await Promise.all(
+    resolvedVideos.map(async ({ video, videoPath }) => {
+      if (signal?.aborted) {
+        throw new Error("Video frame extraction cancelled");
+      }
+      try {
         let videoDuration = video.end - video.start;
 
         // Fallback: if no data-duration/data-end was specified (end is Infinity or 0),

package/src/services/videoFrameInjector.ts

@@ -117,3 +117,203 @@ export function createVideoFrameInjector(
     }
   };
 }
+
+// ── HDR compositing utilities ─────────────────────────────────────────────────
+
+/**
+ * Bounds and transform of a video element, queried from Chrome each frame.
+ * Used by the two-pass HDR compositing pipeline to position native HDR frames.
+ */
+export interface VideoElementBounds {
+  videoId: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  opacity: number;
+  /** CSS transform matrix as a DOMMatrix-compatible string, e.g. "matrix(1,0,0,1,0,0)" */
+  transform: string;
+  zIndex: number;
+  visible: boolean;
+}
+
+/**
+ * Hide specific video elements by ID. Used in Pass 1 of the HDR pipeline so
+ * Chrome screenshots only contain DOM content (text, overlays) with transparent
+ * holes where the HDR videos go.
+ */
+export async function hideVideoElements(page: Page, videoIds: string[]): Promise<void> {
+  if (videoIds.length === 0) return;
+  await page.evaluate((ids: string[]) => {
+    for (const id of ids) {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (el) {
+        el.style.setProperty("visibility", "hidden", "important");
+        el.style.setProperty("opacity", "0", "important");
+        // Also hide the injected render frame image if present
+        const img = document.getElementById(`__render_frame_${id}__`);
+        if (img) img.style.setProperty("visibility", "hidden", "important");
+      }
+    }
+  }, videoIds);
+}
+
+/**
+ * Restore visibility of video elements after a DOM screenshot.
+ */
+export async function showVideoElements(page: Page, videoIds: string[]): Promise<void> {
+  if (videoIds.length === 0) return;
+  await page.evaluate((ids: string[]) => {
+    for (const id of ids) {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (el) {
+        el.style.removeProperty("visibility");
+        el.style.removeProperty("opacity");
+        const img = document.getElementById(`__render_frame_${id}__`);
+        if (img) img.style.removeProperty("visibility");
+      }
+    }
+  }, videoIds);
+}
+
+/**
+ * Query the current bounds, transform, and visibility of video elements.
+ * Called after seeking (so GSAP has moved things) but before the screenshot.
+ */
+export async function queryVideoElementBounds(
+  page: Page,
+  videoIds: string[],
+): Promise<VideoElementBounds[]> {
+  if (videoIds.length === 0) return [];
+  return page.evaluate((ids: string[]): VideoElementBounds[] => {
+    return ids.map((id) => {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (!el) {
+        return {
+          videoId: id,
+          x: 0,
+          y: 0,
+          width: 0,
+          height: 0,
+          opacity: 0,
+          transform: "none",
+          zIndex: 0,
+          visible: false,
+        };
+      }
+      const rect = el.getBoundingClientRect();
+      const style = window.getComputedStyle(el);
+      const zIndex = parseInt(style.zIndex) || 0;
+      const opacity = parseFloat(style.opacity) || 1;
+      const transform = style.transform || "none";
+      const visible =
+        style.visibility !== "hidden" &&
+        style.display !== "none" &&
+        rect.width > 0 &&
+        rect.height > 0;
+      return {
+        videoId: id,
+        x: Math.round(rect.x),
+        y: Math.round(rect.y),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        opacity,
+        transform,
+        zIndex,
+        visible,
+      };
+    });
+  }, videoIds);
+}
+
+/**
+ * Stacking info for a single timed element, used by the z-ordered layer compositor.
+ */
+export interface ElementStackingInfo {
+  id: string;
+  zIndex: number;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  opacity: number;
+  visible: boolean;
+  isHdr: boolean;
+}
+
+/**
+ * Query Chrome for ALL timed elements' stacking context.
+ * Returns z-index, bounds, opacity, and whether each element is a native HDR video.
+ *
+ * Queries every element with `data-start` (not just videos) so the layer compositor
+ * can determine z-ordering between DOM content and HDR video elements.
+ */
+export async function queryElementStacking(
+  page: Page,
+  nativeHdrVideoIds: Set<string>,
+): Promise<ElementStackingInfo[]> {
+  const hdrIds = Array.from(nativeHdrVideoIds);
+  return page.evaluate((hdrIdList: string[]): ElementStackingInfo[] => {
+    const hdrSet = new Set(hdrIdList);
+    const elements = document.querySelectorAll("[data-start]");
+    const results: ElementStackingInfo[] = [];
+
+    // Walk up the DOM to find the effective z-index from the nearest
+    // positioned ancestor with a z-index. CSS z-index only applies to
+    // positioned elements; video elements inside positioned wrappers
+    // inherit the wrapper's stacking context.
+    //
+    // ## Supported subset
+    //
+    // This implementation looks for explicit `z-index` on positioned
+    // (non-static) ancestors. It does NOT detect the CSS stacking contexts
+    // created implicitly by other properties — including `opacity < 1`,
+    // `transform`, `filter`, `will-change`, `isolation: isolate`, and
+    // `mix-blend-mode`. GSAP routinely sets `transform` on wrappers, which
+    // creates an implicit stacking context with auto z-index; an HDR video
+    // inside such a wrapper with no explicit z-index will return the
+    // wrapper-of-the-wrapper's z-index here, potentially reordering layers
+    // incorrectly relative to sibling stacking contexts.
+    //
+    // The workaround is to set explicit `z-index` on the positioned wrapper
+    // when you want it treated as a compositing layer root. This matches
+    // what compositions need to do anyway for deterministic z-ordering.
+    function getEffectiveZIndex(node: Element): number {
+      let current: Element | null = node;
+      while (current) {
+        const cs = window.getComputedStyle(current);
+        const pos = cs.position;
+        const z = parseInt(cs.zIndex);
+        if (!Number.isNaN(z) && pos !== "static") return z;
+        current = current.parentElement;
+      }
+      return 0;
+    }
+
+    for (const el of elements) {
+      const id = el.id;
+      if (!id) continue;
+      const rect = el.getBoundingClientRect();
+      const style = window.getComputedStyle(el);
+      const zIndex = getEffectiveZIndex(el);
+      const opacity = parseFloat(style.opacity) || 1;
+      const visible =
+        style.visibility !== "hidden" &&
+        style.display !== "none" &&
+        rect.width > 0 &&
+        rect.height > 0;
+      results.push({
+        id,
+        zIndex,
+        x: Math.round(rect.x),
+        y: Math.round(rect.y),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        opacity,
+        visible,
+        isHdr: hdrSet.has(id),
+      });
+    }
+    return results;
+  }, hdrIds);
+}