@hyperframes/engine 0.5.0-alpha.9 → 0.5.0

package/src/services/videoFrameExtractor.ts

@@ -7,7 +7,7 @@
 
 import { spawn } from "child_process";
 import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
-import { isAbsolute, join } from "path";
+import { isAbsolute, join, posix, resolve, sep } from "path";
 import { parseHTML } from "linkedom";
 import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
 import {
@@ -230,8 +230,17 @@ export async function extractVideoFramesRange(
   if (isHdr && isMacOS) {
     args.push("-hwaccel", "videotoolbox");
   }
-  if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
-    args.push("-c:v", "libvpx-vp9");
+  // Always force the alpha-aware decoder on codecs that can carry alpha. The
+  // alternative — gating on `metadata.hasAlpha` — relies on tag detection that
+  // has at least three known failure modes: case-sensitivity across ffmpeg
+  // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
+  // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
+  // silently strips alpha during decode and the bug doesn't surface until
+  // the rendered video is missing layers. Codec-based default has no such
+  // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
+  // normally when it isn't.
+  if (codecMayHaveAlpha(metadata.videoCodec)) {
+    args.push("-c:v", decoderForCodec(metadata.videoCodec));
   }
   args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
 
@@ -398,9 +407,31 @@ function resolveSegmentDuration(
   return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
 }
 
+/**
+ * Codecs whose bitstream is allowed to carry an alpha channel. Default the
+ * extraction path to PNG output for these regardless of `metadata.hasAlpha`
+ * so a missed sidecar tag doesn't silently strip transparency. Opaque content
+ * encoded in one of these codecs pays a small file-size cost on the cached
+ * frames but stays correct on the rare case where alpha IS present and the
+ * tag was missed.
+ */
+const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
+
+export function codecMayHaveAlpha(codec: string | undefined): boolean {
+  return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
+}
+
+export function decoderForCodec(codec: string | undefined): string {
+  const c = (codec ?? "").toLowerCase();
+  if (c === "vp9") return "libvpx-vp9";
+  if (c === "vp8") return "libvpx";
+  return c;
+}
+
 function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
   if (requested) return requested;
-  return metadata.hasAlpha ? "png" : "jpg";
+  if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
+  return "jpg";
 }
 
 /**
@@ -459,6 +490,54 @@ async function convertVfrToCfr(
   }
 }
 
+/**
+ * Resolve a relative `<video src>` to a filesystem path the way the browser
+ * resolves it as a URL. Browsers clamp `..` segments at the served origin's
+ * root; `path.join(projectDir, "../assets/foo")` does not. So a sub-comp
+ * `<video src="../assets/foo">` loads in the page (browser clamps to
+ * `<projectDir>/assets/foo`) but the filesystem-side resolver lands at
+ * `<parentOfProjectDir>/assets/foo` — file missing, extraction skipped,
+ * the rendered output shows the video's first frame for the whole clip.
+ *
+ * The clamp covers two escape patterns: leading `..` (`../assets/foo`) AND
+ * mid-path escapes (`assets/../../foo`) that `path.join` collapses past the
+ * project root silently. Both fall back to a project-rooted candidate that
+ * strips traversal from the resolved path.
+ *
+ * Returns the first existing candidate, or the base-dir join on miss so
+ * the caller's `existsSync` check produces a stable error path.
+ */
+export function resolveProjectRelativeSrc(
+  src: string,
+  baseDir: string,
+  compiledDir?: string,
+): string {
+  const fromCompiled = compiledDir ? join(compiledDir, src) : null;
+  const fromBase = join(baseDir, src);
+  const candidates: string[] = [];
+  if (fromCompiled) candidates.push(fromCompiled);
+  candidates.push(fromBase);
+  // If the joined result escapes the project root (either via leading `..`
+  // or mid-path traversal that path.join collapsed past baseDir), retry
+  // with the basename re-anchored at the project root. This mirrors the
+  // browser URL clamp without relying on a particular `..` shape.
+  const baseAbs = resolve(baseDir);
+  const fromBaseAbs = resolve(fromBase);
+  if (!fromBaseAbs.startsWith(baseAbs + sep) && fromBaseAbs !== baseAbs) {
+    // Normalize first (`assets/../../assets/foo.mp4` → `../assets/foo.mp4`)
+    // then strip any remaining leading `..` segments. Stripping `..` from the
+    // raw input would leave dangling siblings (`assets/../../assets/foo`
+    // would become `assets/assets/foo` instead of `assets/foo`).
+    const normalized = posix.normalize(src.replace(/\\/g, "/"));
+    const stripped = normalized.replace(/^(\.\.\/)+/, "");
+    if (stripped && stripped !== src && !stripped.startsWith("..")) {
+      if (compiledDir) candidates.push(join(compiledDir, stripped));
+      candidates.push(join(baseDir, stripped));
+    }
+  }
+  return candidates.find(existsSync) ?? fromBase;
+}
+
 export async function extractAllVideoFrames(
   videos: VideoElement[],
   baseDir: string,
@@ -487,6 +566,9 @@ export async function extractAllVideoFrames(
   // Phase 1: Resolve paths and download remote videos
   const phase1Start = Date.now();
   const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
+  // Dedupe missing-src warnings: a composition with N <video> elements all
+  // pointing at the same broken src should only print one warning, not N.
+  const warnedSrcs = new Set<string>();
   for (const video of videos) {
     if (signal?.aborted) break;
     try {
@@ -496,9 +578,7 @@ export async function extractAllVideoFrames(
       // baseDir and produce duplicated, nonexistent paths
       // (e.g. C:\tmp\hf-vfr-test-X\C:\tmp\hf-vfr-test-X\vfr_screen.mp4).
       if (!isAbsolute(videoPath) && !isHttpUrl(videoPath)) {
-        const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
-        videoPath =
-          fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
+        videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
       }
 
       if (isHttpUrl(videoPath)) {
@@ -508,6 +588,19 @@ export async function extractAllVideoFrames(
       }
 
       if (!existsSync(videoPath)) {
+        // Loud: silent miss leaves the rendered video frozen at frame 0 with
+        // no error in stdout — extremely confusing for authors. Dedupe by
+        // src so 50 broken videos pointing at the same path don't spam.
+        if (!warnedSrcs.has(video.src)) {
+          warnedSrcs.add(video.src);
+          process.stderr.write(
+            `[hyperframes:render] WARNING: video src="${video.src}" ` +
+              `could not be resolved on disk (looked for ${videoPath}). ` +
+              `The rendered output will show this video's first frame for the entire clip duration. ` +
+              `If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
+              `(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
+          );
+        }
         errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
         continue;
       }

package/src/services/videoFrameInjector.ts

@@ -14,7 +14,16 @@ import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotSe
 import { type BeforeCaptureHook } from "./frameCapture.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 
-function createFrameDataUriCache(cacheLimit: number) {
+export interface VideoFrameInjectorOptions extends Partial<
+  Pick<EngineConfig, "frameDataUriCacheLimit">
+> {
+  frameSrcResolver?: (framePath: string) => string | null;
+}
+
+function createFrameSourceCache(
+  cacheLimit: number,
+  frameSrcResolver?: (framePath: string) => string | null,
+) {
   const cache = new Map<string, string>();
   const inFlight = new Map<string, Promise<string>>();
 
@@ -33,6 +42,9 @@ function createFrameDataUriCache(cacheLimit: number) {
   }
 
   async function get(framePath: string): Promise<string> {
+    const servedSrc = frameSrcResolver?.(framePath);
+    if (servedSrc) return servedSrc;
+
     const cached = cache.get(framePath);
     if (cached) {
       remember(framePath, cached);
@@ -67,7 +79,7 @@ function createFrameDataUriCache(cacheLimit: number) {
  */
 export function createVideoFrameInjector(
   frameLookup: FrameLookupTable | null,
-  config?: Partial<Pick<EngineConfig, "frameDataUriCacheLimit">>,
+  config?: VideoFrameInjectorOptions,
 ): BeforeCaptureHook | null {
   if (!frameLookup) return null;
 
@@ -75,7 +87,7 @@ export function createVideoFrameInjector(
     32,
     config?.frameDataUriCacheLimit ?? DEFAULT_CONFIG.frameDataUriCacheLimit,
   );
-  const frameCache = createFrameDataUriCache(cacheLimit);
+  const frameCache = createFrameSourceCache(cacheLimit, config?.frameSrcResolver);
   const lastInjectedFrameByVideo = new Map<string, number>();
 
   return async (page: Page, time: number) => {

package/src/types.ts

@@ -85,18 +85,40 @@ export interface CaptureOptions {
   format?: "jpeg" | "png";
   quality?: number;
   deviceScaleFactor?: number;
+  /**
+   * FFmpeg-probed intrinsic dimensions for videos whose frames are injected
+   * out-of-band. Applied before the readiness wait so layout that depends on
+   * video aspect ratio (e.g. `height:auto`) stays stable even if Chromium never
+   * loads native metadata.
+   */
+  videoMetadataHints?: readonly CaptureVideoMetadataHint[];
   /**
    * Video element IDs to exclude from the in-page readiness check that waits
    * for `video.readyState >= 1` before capture starts.
    *
-   * Use for videos whose frames are supplied out-of-band (e.g. native HDR
-   * frame extraction via ffmpeg). The DOM `<video>` element is then only
-   * needed for layout (`getBoundingClientRect` / `offsetWidth`), which works
-   * at `readyState=0`. Without this, codecs that headless Chromium can't
-   * decode (HEVC on Linux `headless-shell`) cause a fatal timeout even
-   * though we never asked the browser to play the video.
+   * Use for videos whose frames are supplied out-of-band, including standard
+   * FFmpeg frame injection and native HDR extraction. Pair with
+   * `videoMetadataHints` for any skipped video whose CSS layout may depend on
+   * intrinsic media dimensions.
    */
   skipReadinessVideoIds?: readonly string[];
+  /**
+   * Render-time variable overrides for the composition. The engine injects
+   * these as `window.__hfVariables` via `evaluateOnNewDocument` before any
+   * page script runs, so the runtime helper `getVariables()` returns the
+   * merged result of declared defaults (`data-composition-variables`) and
+   * these overrides on its first call.
+   *
+   * The CLI populates this from `--variables '<json>'` /
+   * `--variables-file <path>`. Must be a JSON-serializable plain object.
+   */
+  variables?: Record<string, unknown>;
+}
+
+export interface CaptureVideoMetadataHint {
+  id: string;
+  width: number;
+  height: number;
 }
 
 export interface CaptureResult {

package/src/utils/alphaBlit.test.ts

@@ -511,6 +511,16 @@ describe("blitRgba8OverRgb48le", () => {
     expect(canvas.readUInt16LE(4)).toBe(0);
   });
 
+  it("fully opaque DOM with srgb transfer expands 8-bit channels to 16-bit SDR", () => {
+    const canvas = makeHdrFrame(1, 1, 10000, 20000, 30000);
+    const dom = makeDomRgba(1, 1, 255, 128, 1, 255);
+    blitRgba8OverRgb48le(dom, canvas, 1, 1, "srgb");
+
+    expect(canvas.readUInt16LE(0)).toBe(65535);
+    expect(canvas.readUInt16LE(2)).toBe(128 * 257);
+    expect(canvas.readUInt16LE(4)).toBe(257);
+  });
+
   it("sRGB→HLG: black stays black, white stays white", () => {
     const canvasBlack = makeHdrFrame(1, 1, 0, 0, 0);
     const domBlack = makeDomRgba(1, 1, 0, 0, 0, 255);

package/src/utils/alphaBlit.ts

@@ -249,7 +249,7 @@ export function decodePngToRgb48le(buf: Buffer): { width: number; height: number
  * bt2020). For neutral/near-neutral content (text, UI) the gamut difference
  * is negligible.
  */
-function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
+function buildSrgbToSignalLut(transfer: "hlg" | "pq" | "srgb"): Uint16Array {
   const lut = new Uint16Array(256);
 
   // HLG OETF constants (Rec. 2100)
@@ -267,6 +267,11 @@ function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
   const sdrNits = 203.0;
 
   for (let i = 0; i < 256; i++) {
+    if (transfer === "srgb") {
+      lut[i] = i * 257;
+      continue;
+    }
+
     // sRGB EOTF: signal → linear (range 0–1, relative to SDR white)
     const v = i / 255;
     const linear = v <= 0.04045 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4);
@@ -288,12 +293,15 @@ function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
   return lut;
 }
 
-const SRGB_TO_HLG = buildSrgbToHdrLut("hlg");
-const SRGB_TO_PQ = buildSrgbToHdrLut("pq");
+const SRGB_TO_SRGB_16 = buildSrgbToSignalLut("srgb");
+const SRGB_TO_HLG = buildSrgbToSignalLut("hlg");
+const SRGB_TO_PQ = buildSrgbToSignalLut("pq");
 
 /** Select the correct sRGB→HDR LUT for the given transfer function. */
-function getSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array {
-  return transfer === "pq" ? SRGB_TO_PQ : SRGB_TO_HLG;
+function getSrgbToSignalLut(transfer: "hlg" | "pq" | "srgb"): Uint16Array {
+  if (transfer === "pq") return SRGB_TO_PQ;
+  if (transfer === "hlg") return SRGB_TO_HLG;
+  return SRGB_TO_SRGB_16;
 }
 
 // ── Alpha compositing ─────────────────────────────────────────────────────────
@@ -317,10 +325,10 @@ export function blitRgba8OverRgb48le(
   canvas: Buffer,
   width: number,
   height: number,
-  transfer: "hlg" | "pq" = "hlg",
+  transfer: "hlg" | "pq" | "srgb" = "hlg",
 ): void {
   const pixelCount = width * height;
-  const lut = getSrgbToHdrLut(transfer);
+  const lut = getSrgbToSignalLut(transfer);
 
   for (let i = 0; i < pixelCount; i++) {
     const da = domRgba[i * 4 + 3] ?? 0;

package/src/utils/ffprobe.test.ts

@@ -225,6 +225,46 @@ describe("ffprobe missing-binary fallback", () => {
     expect(meta.hasAlpha).toBe(true);
   });
 
+  // Regression: newer libavformat builds (and the output of `hyperframes
+  // remove-background` itself) write the VP9-alpha sidecar tag as
+  // `ALPHA_MODE` (uppercase). The lowercase-only check classified those
+  // files as having no alpha, the producer extracted them as JPGs, and
+  // the injected <img> overlays were fully opaque rectangles that hid
+  // every static element below them on the z-stack. The bug was silent —
+  // studio preview rendered correctly via native <video> playback while
+  // production renders covered headlines and captions with the avatar.
+  it("extractMediaMetadata detects ALPHA_MODE (uppercase) streams from newer ffmpeg builds", async () => {
+    const { spawn } = createSpawnSpy([
+      {
+        kind: "exit",
+        code: 0,
+        stdout: JSON.stringify({
+          streams: [
+            {
+              codec_type: "video",
+              codec_name: "vp9",
+              width: 320,
+              height: 180,
+              r_frame_rate: "30/1",
+              avg_frame_rate: "30/1",
+              pix_fmt: "yuv420p",
+              tags: { ALPHA_MODE: "1" },
+            },
+          ],
+          format: { duration: "1.5" },
+        }),
+      },
+    ]);
+    vi.resetModules();
+    vi.doMock("child_process", () => ({ spawn }));
+
+    const { extractMediaMetadata: extractMediaMetadataMocked } = await import("./ffprobe.js");
+    const meta = await extractMediaMetadataMocked("/tmp/alpha-uppercase.webm");
+
+    expect(meta.videoCodec).toBe("vp9");
+    expect(meta.hasAlpha).toBe(true);
+  });
+
   it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
     const { spawn } = createSpawnSpy([{ kind: "missing" }]);
     vi.resetModules();

package/src/utils/ffprobe.ts

@@ -203,6 +203,21 @@ function extractStillImageMetadata(filePath: string): StillImageMetadata | null
   }
 }
 
+/**
+ * Read an ffprobe tag case-insensitively. ffmpeg/libavformat versions disagree
+ * on tag casing — VP9 alpha is `alpha_mode` in older builds and `ALPHA_MODE`
+ * in newer ones; HDR tags vary similarly. Use this for any sidecar tag where
+ * you want to be resilient across muxer versions.
+ */
+function readTagCI(tags: Record<string, string | undefined> | undefined, name: string): string {
+  if (!tags) return "";
+  const target = name.toLowerCase();
+  for (const [key, value] of Object.entries(tags)) {
+    if (key.toLowerCase() === target && typeof value === "string") return value;
+  }
+  return "";
+}
+
 function parseFrameRate(frameRateStr: string | undefined): number {
   if (!frameRateStr) return 0;
   const parts = frameRateStr.split("/");
@@ -277,7 +292,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
         : null;
     const colorSpace = ffprobeColorSpace ?? stillImageMeta?.colorSpace ?? null;
     const pixelFormat = videoStream.pix_fmt || "";
-    const alphaMode = videoStream.tags?.alpha_mode || "";
+    const alphaMode = readTagCI(videoStream.tags, "alpha_mode");
     const hasAlpha =
       /(^|[^a-z])yuva|rgba|argb|bgra|gbrap|gray[a-z0-9]*a/i.test(pixelFormat) || alphaMode === "1";
 

package/src/utils/htmlTemplate.ts

@@ -1,12 +1,4 @@
-import { parseHTML } from "linkedom";
-
-function parseHTMLContent(html: string): Document {
-  const trimmed = html.trimStart().toLowerCase();
-  if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html")) {
-    return parseHTML(html).document;
-  }
-  return parseHTML(`<!DOCTYPE html><html><head></head><body>${html}</body></html>`).document;
-}
+import { parseHTMLContent } from "@hyperframes/core/compiler";
 
 function getSingleMeaningfulChild(container: Element): Element | null {
   let child: Element | null = null;

package/src/utils/parityContract.ts

@@ -1,35 +1 @@
-export const MEDIA_VISUAL_STYLE_PROPERTIES = [
-  "width",
-  "height",
-  "top",
-  "left",
-  "right",
-  "bottom",
-  "inset",
-  "object-fit",
-  "object-position",
-  "z-index",
-  "opacity",
-  "visibility",
-  "filter",
-  "mix-blend-mode",
-  "backdrop-filter",
-  "border-radius",
-  "overflow",
-  "clip-path",
-  "mask",
-  "mask-image",
-  "mask-size",
-  "mask-position",
-  "mask-repeat",
-  "transform",
-  "transform-origin",
-  "box-sizing",
-] as const;
-
-export function quantizeTimeToFrame(timeSeconds: number, fps: number): number {
-  const safeFps = Number.isFinite(fps) && fps > 0 ? fps : 30;
-  const safeTime = Number.isFinite(timeSeconds) && timeSeconds > 0 ? timeSeconds : 0;
-  const frameIndex = Math.floor(safeTime * safeFps + 1e-9);
-  return frameIndex / safeFps;
-}
+export { MEDIA_VISUAL_STYLE_PROPERTIES, quantizeTimeToFrame } from "@hyperframes/core";