@hyperframes/engine 0.4.6 → 0.4.8

package/src/services/videoFrameExtractor.ts

@@ -10,7 +10,9 @@ import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
 import { join } from "path";
 import { parseHTML } from "linkedom";
 import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js";
+import { isHdrColorSpace as isHdrColorSpaceUtil } from "../utils/hdr.js";
 import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
+import { runFfmpeg } from "../utils/runFfmpeg.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 
 export interface VideoElement {
@@ -114,18 +116,28 @@ export async function extractVideoFramesRange(
   const framePattern = `frame_%05d.${format}`;
   const outputPattern = join(videoOutputDir, framePattern);
 
-  const args: string[] = [
-    "-ss",
-    String(startTime),
-    "-i",
-    videoPath,
-    "-t",
-    String(duration),
-    "-vf",
-    `fps=${fps}`,
-    "-q:v",
-    format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0",
-  ];
+  // When extracting from HDR source, tone-map to SDR in FFmpeg rather than
+  // letting Chrome's uncontrollable tone-mapper handle it (which washes out).
+  // macOS: VideoToolbox hardware decoder does HDR→SDR natively on Apple Silicon.
+  // Linux: zscale filter (when available) or colorspace filter as fallback.
+  const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
+  const isMacOS = process.platform === "darwin";
+
+  const args: string[] = [];
+  if (isHdr && isMacOS) {
+    args.push("-hwaccel", "videotoolbox");
+  }
+  args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
+
+  const vfFilters: string[] = [];
+  if (isHdr && isMacOS) {
+    // VideoToolbox tone-maps during decode; force output to bt709 SDR format
+    vfFilters.push("format=nv12");
+  }
+  vfFilters.push(`fps=${fps}`);
+  args.push("-vf", vfFilters.join(","));
+
+  args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0");
   if (format === "png") args.push("-compression_level", "6");
   args.push("-y", outputPattern);
 
@@ -195,6 +207,53 @@ export async function extractVideoFramesRange(
   });
 }
 
+/**
+ * Convert an SDR video to HDR color space (HLG / BT.2020) so it can be
+ * composited alongside HDR content without looking washed out.
+ *
+ * Uses zscale for color space conversion with a nominal peak luminance of
+ * 600 nits — high enough that SDR content doesn't appear too dark next to
+ * HDR, matching the approach used by HeyGen's Rio pipeline.
+ */
+async function convertSdrToHdr(
+  inputPath: string,
+  outputPath: string,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<void> {
+  const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+
+  const args = [
+    "-i",
+    inputPath,
+    "-vf",
+    "colorspace=all=bt2020:iall=bt709:range=tv",
+    "-color_primaries",
+    "bt2020",
+    "-color_trc",
+    "arib-std-b67",
+    "-colorspace",
+    "bt2020nc",
+    "-c:v",
+    "libx264",
+    "-preset",
+    "fast",
+    "-crf",
+    "16",
+    "-c:a",
+    "copy",
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout });
+  if (!result.success) {
+    throw new Error(
+      `SDR→HDR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`,
+    );
+  }
+}
+
 export async function extractAllVideoFrames(
   videos: VideoElement[],
   baseDir: string,
@@ -208,30 +267,75 @@ export async function extractAllVideoFrames(
   const errors: Array<{ videoId: string; error: string }> = [];
   let totalFramesExtracted = 0;
 
-  // Process videos in parallel for better performance
-  const results = await Promise.all(
-    videos.map(async (video) => {
-      if (signal?.aborted) {
-        throw new Error("Video frame extraction cancelled");
+  // Phase 1: Resolve paths and download remote videos
+  const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
+  for (const video of videos) {
+    if (signal?.aborted) break;
+    try {
+      let videoPath = video.src;
+      if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
+        const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
+        videoPath =
+          fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
       }
-      try {
-        let videoPath = video.src;
-        if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
-          const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
-          videoPath =
-            fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
-        }
 
-        if (isHttpUrl(videoPath)) {
-          const downloadDir = join(options.outputDir, "_downloads");
-          mkdirSync(downloadDir, { recursive: true });
-          videoPath = await downloadToTemp(videoPath, downloadDir);
-        }
+      if (isHttpUrl(videoPath)) {
+        const downloadDir = join(options.outputDir, "_downloads");
+        mkdirSync(downloadDir, { recursive: true });
+        videoPath = await downloadToTemp(videoPath, downloadDir);
+      }
+
+      if (!existsSync(videoPath)) {
+        errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
+        continue;
+      }
+      resolvedVideos.push({ video, videoPath });
+    } catch (err) {
+      errors.push({ videoId: video.id, error: err instanceof Error ? err.message : String(err) });
+    }
+  }
 
-        if (!existsSync(videoPath)) {
-          return { error: { videoId: video.id, error: `Video file not found: ${videoPath}` } };
+  // Phase 2: Probe color spaces and normalize if mixed HDR/SDR
+  const videoColorSpaces = await Promise.all(
+    resolvedVideos.map(async ({ videoPath }) => {
+      const metadata = await extractVideoMetadata(videoPath);
+      return metadata.colorSpace;
+    }),
+  );
+
+  const hasAnyHdr = videoColorSpaces.some(isHdrColorSpaceUtil);
+  if (hasAnyHdr) {
+    const convertDir = join(options.outputDir, "_hdr_normalized");
+    mkdirSync(convertDir, { recursive: true });
+
+    for (let i = 0; i < resolvedVideos.length; i++) {
+      if (signal?.aborted) break;
+      const cs = videoColorSpaces[i] ?? null;
+      if (!isHdrColorSpaceUtil(cs)) {
+        // SDR video in a mixed timeline — convert to HDR color space
+        const entry = resolvedVideos[i];
+        if (!entry) continue;
+        const convertedPath = join(convertDir, `${entry.video.id}_hdr.mp4`);
+        try {
+          await convertSdrToHdr(entry.videoPath, convertedPath, signal, config);
+          entry.videoPath = convertedPath;
+        } catch (err) {
+          errors.push({
+            videoId: entry.video.id,
+            error: `SDR→HDR conversion failed: ${err instanceof Error ? err.message : String(err)}`,
+          });
         }
+      }
+    }
+  }
 
+  // Phase 3: Extract frames (parallel)
+  const results = await Promise.all(
+    resolvedVideos.map(async ({ video, videoPath }) => {
+      if (signal?.aborted) {
+        throw new Error("Video frame extraction cancelled");
+      }
+      try {
         let videoDuration = video.end - video.start;
 
         // Fallback: if no data-duration/data-end was specified (end is Infinity or 0),

package/src/services/videoFrameInjector.ts

@@ -117,3 +117,345 @@ export function createVideoFrameInjector(
     }
   };
 }
+
+// ── HDR compositing utilities ─────────────────────────────────────────────────
+
+/**
+ * Bounds and transform of a video element, queried from Chrome each frame.
+ * Used by the two-pass HDR compositing pipeline to position native HDR frames.
+ */
+export interface VideoElementBounds {
+  videoId: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  opacity: number;
+  /** CSS transform matrix as a DOMMatrix-compatible string, e.g. "matrix(1,0,0,1,0,0)" */
+  transform: string;
+  zIndex: number;
+  visible: boolean;
+}
+
+/**
+ * Hide specific video elements by ID. Used in Pass 1 of the HDR pipeline so
+ * Chrome screenshots only contain DOM content (text, overlays) with transparent
+ * holes where the HDR videos go.
+ */
+export async function hideVideoElements(page: Page, videoIds: string[]): Promise<void> {
+  if (videoIds.length === 0) return;
+  await page.evaluate((ids: string[]) => {
+    for (const id of ids) {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (el) {
+        el.style.setProperty("visibility", "hidden", "important");
+        el.style.setProperty("opacity", "0", "important");
+        // Also hide the injected render frame image if present
+        const img = document.getElementById(`__render_frame_${id}__`);
+        if (img) img.style.setProperty("visibility", "hidden", "important");
+      }
+    }
+  }, videoIds);
+}
+
+/**
+ * Restore visibility of video elements after a DOM screenshot.
+ */
+export async function showVideoElements(page: Page, videoIds: string[]): Promise<void> {
+  if (videoIds.length === 0) return;
+  await page.evaluate((ids: string[]) => {
+    for (const id of ids) {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (el) {
+        el.style.removeProperty("visibility");
+        el.style.removeProperty("opacity");
+        const img = document.getElementById(`__render_frame_${id}__`);
+        if (img) img.style.removeProperty("visibility");
+      }
+    }
+  }, videoIds);
+}
+
+/**
+ * Query the current bounds, transform, and visibility of video elements.
+ * Called after seeking (so GSAP has moved things) but before the screenshot.
+ */
+export async function queryVideoElementBounds(
+  page: Page,
+  videoIds: string[],
+): Promise<VideoElementBounds[]> {
+  if (videoIds.length === 0) return [];
+  return page.evaluate((ids: string[]): VideoElementBounds[] => {
+    return ids.map((id) => {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (!el) {
+        return {
+          videoId: id,
+          x: 0,
+          y: 0,
+          width: 0,
+          height: 0,
+          opacity: 0,
+          transform: "none",
+          zIndex: 0,
+          visible: false,
+        };
+      }
+      const rect = el.getBoundingClientRect();
+      const style = window.getComputedStyle(el);
+      const zIndex = parseInt(style.zIndex) || 0;
+      const opacity = parseFloat(style.opacity) || 1;
+      const transform = style.transform || "none";
+      const visible =
+        style.visibility !== "hidden" &&
+        style.display !== "none" &&
+        rect.width > 0 &&
+        rect.height > 0;
+      return {
+        videoId: id,
+        x: Math.round(rect.x),
+        y: Math.round(rect.y),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        opacity,
+        transform,
+        zIndex,
+        visible,
+      };
+    });
+  }, videoIds);
+}
+
+/**
+ * Stacking info for a single timed element, used by the z-ordered layer compositor.
+ */
+export interface ElementStackingInfo {
+  id: string;
+  zIndex: number;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  /** Layout dimensions before CSS transforms (offsetWidth/offsetHeight). */
+  layoutWidth: number;
+  layoutHeight: number;
+  opacity: number;
+  visible: boolean;
+  isHdr: boolean;
+  transform: string; // CSS transform matrix string, e.g. "matrix(1,0,0,1,0,0)" or "none"
+  borderRadius: [number, number, number, number]; // [tl, tr, br, bl] in CSS px from nearest clipping ancestor
+}
+
+/**
+ * Query Chrome for ALL timed elements' stacking context.
+ * Returns z-index, bounds, opacity, and whether each element is a native HDR video.
+ *
+ * Queries every element with `data-start` (not just videos) so the layer compositor
+ * can determine z-ordering between DOM content and HDR video elements.
+ */
+export async function queryElementStacking(
+  page: Page,
+  nativeHdrVideoIds: Set<string>,
+): Promise<ElementStackingInfo[]> {
+  const hdrIds = Array.from(nativeHdrVideoIds);
+  return page.evaluate((hdrIdList: string[]): ElementStackingInfo[] => {
+    const hdrSet = new Set(hdrIdList);
+    const elements = document.querySelectorAll("[data-start]");
+    const results: ElementStackingInfo[] = [];
+
+    // Walk up the DOM to find the effective z-index from the nearest
+    // positioned ancestor with a z-index. CSS z-index only applies to
+    // positioned elements; video elements inside positioned wrappers
+    // inherit the wrapper's stacking context.
+    //
+    // ## Supported subset
+    //
+    // This implementation looks for explicit `z-index` on positioned
+    // (non-static) ancestors. It does NOT detect the CSS stacking contexts
+    // created implicitly by other properties — including `opacity < 1`,
+    // `transform`, `filter`, `will-change`, `isolation: isolate`, and
+    // `mix-blend-mode`. GSAP routinely sets `transform` on wrappers, which
+    // creates an implicit stacking context with auto z-index; an HDR video
+    // inside such a wrapper with no explicit z-index will return the
+    // wrapper-of-the-wrapper's z-index here, potentially reordering layers
+    // incorrectly relative to sibling stacking contexts.
+    //
+    // The workaround is to set explicit `z-index` on the positioned wrapper
+    // when you want it treated as a compositing layer root. This matches
+    // what compositions need to do anyway for deterministic z-ordering.
+    function getEffectiveZIndex(node: Element): number {
+      let current: Element | null = node;
+      while (current) {
+        const cs = window.getComputedStyle(current);
+        const pos = cs.position;
+        const z = parseInt(cs.zIndex);
+        if (!Number.isNaN(z) && pos !== "static") return z;
+        current = current.parentElement;
+      }
+      return 0;
+    }
+
+    // Find border-radius that clips the element. Replaced elements like <video>
+    // clip to their own border-radius; ancestors need overflow !== visible.
+    function getEffectiveBorderRadius(node: Element): [number, number, number, number] {
+      // Resolve a CSS border-radius value to pixels. Chrome's getComputedStyle
+      // returns percentages as-is (e.g. "50%"), not resolved to px.
+      // Uses offsetWidth/offsetHeight (layout dimensions before CSS transforms)
+      // because CSS resolves percentages against the padding box, not the
+      // transformed bounding box.
+      function resolveRadius(value: string, el: Element): number {
+        if (value.includes("%")) {
+          const pct = parseFloat(value) / 100;
+          const htmlEl = el as HTMLElement;
+          const w = htmlEl.offsetWidth || 0;
+          const h = htmlEl.offsetHeight || 0;
+          return pct * Math.min(w, h);
+        }
+        return parseFloat(value) || 0;
+      }
+
+      // Check element itself (replaced elements clip to own border-radius)
+      const selfCs = window.getComputedStyle(node);
+      const selfRadii: [number, number, number, number] = [
+        resolveRadius(selfCs.borderTopLeftRadius, node),
+        resolveRadius(selfCs.borderTopRightRadius, node),
+        resolveRadius(selfCs.borderBottomRightRadius, node),
+        resolveRadius(selfCs.borderBottomLeftRadius, node),
+      ];
+      if (selfRadii[0] > 0 || selfRadii[1] > 0 || selfRadii[2] > 0 || selfRadii[3] > 0) {
+        return selfRadii;
+      }
+
+      // Walk ancestors looking for clipping container
+      let current: Element | null = node.parentElement;
+      while (current) {
+        const cs = window.getComputedStyle(current);
+        if (cs.overflow !== "visible") {
+          const tl = resolveRadius(cs.borderTopLeftRadius, current);
+          const tr = resolveRadius(cs.borderTopRightRadius, current);
+          const brr = resolveRadius(cs.borderBottomRightRadius, current);
+          const bl = resolveRadius(cs.borderBottomLeftRadius, current);
+          if (tl > 0 || tr > 0 || brr > 0 || bl > 0) {
+            return [tl, tr, brr, bl];
+          }
+        }
+        current = current.parentElement;
+      }
+      return [0, 0, 0, 0];
+    }
+
+    // Walk up the DOM multiplying each ancestor's opacity. GSAP animates
+    // opacity on wrapper divs, not directly on the video element, so the
+    // element's own opacity is often 1.0. Multiplying ancestors gives the
+    // true effective opacity.
+    function getEffectiveOpacity(node: Element): number {
+      let opacity = 1;
+      let current: Element | null = node;
+      while (current) {
+        const cs = window.getComputedStyle(current);
+        const val = parseFloat(cs.opacity);
+        // Note: `val || 1` would turn opacity:0 into 1 (0 is falsy)
+        opacity *= Number.isNaN(val) ? 1 : val;
+        current = current.parentElement;
+      }
+      return opacity;
+    }
+
+    // Compute the full CSS transform matrix from element-local coords to
+    // viewport coords by walking the offsetParent chain and accumulating
+    // position offsets + CSS transforms. This correctly handles GSAP
+    // animations on wrapper divs (rotation, scale) that getBoundingClientRect
+    // conflates into an axis-aligned bounding box.
+    function getViewportMatrix(node: Element): string {
+      const chain: HTMLElement[] = [];
+      let current: Element | null = node;
+      while (current instanceof HTMLElement) {
+        chain.push(current);
+        const next: Element | null =
+          (current.offsetParent as Element | null) ?? current.parentElement;
+        if (next === current) break;
+        current = next;
+      }
+      let mat = new DOMMatrix();
+      for (let i = chain.length - 1; i >= 0; i--) {
+        const htmlEl = chain[i];
+        if (!htmlEl) continue;
+        mat = mat.translate(htmlEl.offsetLeft, htmlEl.offsetTop);
+        const cs = window.getComputedStyle(htmlEl);
+        if (cs.transform && cs.transform !== "none") {
+          const origin = cs.transformOrigin.split(" ");
+          const ox = resolveLength(origin[0] ?? "0", htmlEl.offsetWidth);
+          const oy = resolveLength(origin[1] ?? "0", htmlEl.offsetHeight);
+          try {
+            const t = new DOMMatrix(cs.transform);
+            if (
+              Number.isFinite(t.a) &&
+              Number.isFinite(t.b) &&
+              Number.isFinite(t.c) &&
+              Number.isFinite(t.d) &&
+              Number.isFinite(t.e) &&
+              Number.isFinite(t.f)
+            ) {
+              mat = mat.translate(ox, oy).multiply(t).translate(-ox, -oy);
+            }
+          } catch {
+            // DOMMatrix constructor throws on malformed input — skip ancestor.
+          }
+        }
+      }
+      return mat.toString();
+    }
+
+    function resolveLength(value: string, basis: number): number {
+      if (value.endsWith("%")) {
+        const pct = parseFloat(value) / 100;
+        return Number.isFinite(pct) ? pct * basis : 0;
+      }
+      const n = parseFloat(value);
+      return Number.isFinite(n) ? n : 0;
+    }
+
+    for (const el of elements) {
+      const id = el.id;
+      if (!id) continue;
+      const rect = el.getBoundingClientRect();
+      const style = window.getComputedStyle(el);
+      const zIndex = getEffectiveZIndex(el);
+      // For HDR video elements, the frame injector sets `opacity: 0 !important`
+      // on the element itself. Start the opacity walk from the parent to get the
+      // real GSAP-animated opacity from wrapper divs.
+      const isHdrEl = hdrSet.has(id);
+      const opacityStartNode = isHdrEl ? el.parentElement : el;
+      const opacity = opacityStartNode ? getEffectiveOpacity(opacityStartNode) : 1;
+      const visible =
+        style.visibility !== "hidden" &&
+        style.display !== "none" &&
+        rect.width > 0 &&
+        rect.height > 0;
+      // offsetWidth/offsetHeight only exist on HTMLElement (not on
+      // SVGElement, MathMLElement, etc.). Fall back to the bounding rect
+      // dimensions for non-HTML elements so callers always get sensible
+      // layout numbers.
+      const htmlEl = el instanceof HTMLElement ? el : null;
+      results.push({
+        id,
+        zIndex,
+        x: Math.round(rect.x),
+        y: Math.round(rect.y),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        layoutWidth: htmlEl?.offsetWidth || Math.round(rect.width),
+        layoutHeight: htmlEl?.offsetHeight || Math.round(rect.height),
+        opacity,
+        visible,
+        isHdr: hdrSet.has(id),
+        // For HDR elements, use the full accumulated viewport matrix so the
+        // affine blit can apply rotation/scale/translate properly. For DOM
+        // elements, the element-level transform is sufficient for reference.
+        transform: isHdrEl ? getViewportMatrix(el) : style.transform || "none",
+        borderRadius: isHdrEl ? getEffectiveBorderRadius(el) : [0, 0, 0, 0],
+      });
+    }
+    return results;
+  }, hdrIds);
+}

package/src/types.ts

@@ -31,6 +31,29 @@ export interface HfMediaElement {
   hasAudio?: boolean;
 }
 
+/**
+ * Metadata for a shader transition between two scenes.
+ *
+ * Compositions using @hyperframes/shader-transitions populate
+ * `window.__hf.transitions` with one entry per transition so the
+ * producer can pre-compute scene ranges, capture per-scene buffers,
+ * and apply the transition in HDR-aware compositing.
+ */
+export interface HfTransitionMeta {
+  /** Time the transition starts (seconds) */
+  time: number;
+  /** Transition duration (seconds) */
+  duration: number;
+  /** Shader identifier (e.g. "fade", "wipe") */
+  shader: string;
+  /** GSAP easing string (e.g. "power2.inOut") */
+  ease: string;
+  /** Scene id the transition starts from */
+  fromScene: string;
+  /** Scene id the transition ends on */
+  toScene: string;
+}
+
 /**
  * The seek protocol. The only contract between the engine and a page.
  *
@@ -42,6 +65,15 @@ export interface HfMediaElement {
  * GSAP, Framer Motion, CSS animations, Three.js — anything works as long
  * as `seek()` produces deterministic visual output for a given time.
  */
+export interface HfTransitionMeta {
+  time: number;
+  duration: number;
+  shader: string;
+  ease: string;
+  fromScene: string;
+  toScene: string;
+}
+
 export interface HfProtocol {
   /** Total duration of the composition in seconds */
   duration: number;
@@ -49,6 +81,8 @@ export interface HfProtocol {
   seek(time: number): void;
   /** Optional: media elements the engine should handle */
   media?: HfMediaElement[];
+  /** Optional: shader transition metadata, populated by @hyperframes/shader-transitions */
+  transitions?: HfTransitionMeta[];
 }
 
 // ── Capture Types ──────────────────────────────────────────────────────────────