@hyperframes/engine 0.4.44 → 0.4.45

package/src/services/videoFrameExtractor.test.ts

@@ -1,5 +1,13 @@
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
 import { createHash } from "node:crypto";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
@@ -9,6 +17,9 @@ import {
   parseImageElements,
   extractAllVideoFrames,
   createFrameLookupTable,
+  resolveProjectRelativeSrc,
+  codecMayHaveAlpha,
+  decoderForCodec,
   type VideoElement,
   type ExtractedFrames,
 } from "./videoFrameExtractor.js";
@@ -23,6 +34,111 @@ import { runFfmpeg } from "../utils/runFfmpeg.js";
 // synthesized VFR fixture.
 const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0;
 
+// Codec-based alpha defaulting replaces tag-based detection (the
+// alpha_mode/ALPHA_MODE case bug — see ffprobe.test.ts for the regression
+// pin on that). The extractor uses these helpers for two decisions:
+//   1. whether to force the alpha-aware decoder (libvpx-vp9 for VP9, libvpx
+//      for VP8)
+//   2. whether to default the cached frame format to PNG (with alpha) vs JPG
+// The "default to capable" trade is small file-size growth on opaque VP9
+// content for correctness on alpha-having content even when the sidecar tag
+// is missing or muxed with the wrong case.
+describe("codec alpha capability", () => {
+  it("flags VP9, VP8, and ProRes as alpha-capable", () => {
+    expect(codecMayHaveAlpha("vp9")).toBe(true);
+    expect(codecMayHaveAlpha("VP9")).toBe(true);
+    expect(codecMayHaveAlpha("vp8")).toBe(true);
+    expect(codecMayHaveAlpha("prores")).toBe(true);
+  });
+
+  it("does not flag h264 / h265 / mpeg4 (no alpha in their bitstreams)", () => {
+    expect(codecMayHaveAlpha("h264")).toBe(false);
+    expect(codecMayHaveAlpha("h265")).toBe(false);
+    expect(codecMayHaveAlpha("hevc")).toBe(false);
+    expect(codecMayHaveAlpha("mpeg4")).toBe(false);
+  });
+
+  it("treats undefined / empty input as non-alpha", () => {
+    expect(codecMayHaveAlpha(undefined)).toBe(false);
+    expect(codecMayHaveAlpha("")).toBe(false);
+  });
+
+  it("returns the alpha-aware decoder name for VP9 and VP8", () => {
+    expect(decoderForCodec("vp9")).toBe("libvpx-vp9");
+    expect(decoderForCodec("VP9")).toBe("libvpx-vp9");
+    expect(decoderForCodec("vp8")).toBe("libvpx");
+  });
+});
+
+// Regression: a long-standing footgun where `<video src="../assets/foo">`
+// inside a sub-composition silently dropped the video from extraction. The
+// browser's URL resolver clamps `..` at the served origin's root (so the
+// page renders fine in the studio), but `path.join(projectDir, "../assets/foo")`
+// normalizes to <parentOfProjectDir>/assets/foo, which doesn't exist —
+// extraction skipped, no frame injection, rendered output shows the video's
+// first decoded frame for the whole clip duration. The resolver now mirrors
+// browser semantics by clamping any traversal that escapes the project root.
+describe("resolveProjectRelativeSrc — sub-composition path clamping", () => {
+  let tmp: string;
+
+  beforeAll(() => {
+    tmp = mkdtempSync(join(tmpdir(), "hf-resolver-"));
+    mkdirSync(join(tmp, "project", "assets"), { recursive: true });
+    writeFileSync(join(tmp, "project", "assets", "foo.mp4"), "");
+  });
+  afterAll(() => {
+    rmSync(tmp, { recursive: true, force: true });
+  });
+
+  it("returns the literal join when the file exists at projectDir/src", () => {
+    const projectDir = join(tmp, "project");
+    expect(resolveProjectRelativeSrc("assets/foo.mp4", projectDir)).toBe(
+      join(projectDir, "assets/foo.mp4"),
+    );
+  });
+
+  it("clamps a leading `../` so `../assets/foo.mp4` resolves to assets/foo.mp4", () => {
+    const projectDir = join(tmp, "project");
+    expect(resolveProjectRelativeSrc("../assets/foo.mp4", projectDir)).toBe(
+      join(projectDir, "assets/foo.mp4"),
+    );
+  });
+
+  it("clamps multiple leading `../../../` segments", () => {
+    const projectDir = join(tmp, "project");
+    expect(resolveProjectRelativeSrc("../../../assets/foo.mp4", projectDir)).toBe(
+      join(projectDir, "assets/foo.mp4"),
+    );
+  });
+
+  it("clamps mid-path traversal that escapes baseDir (not just leading `..`)", () => {
+    // `assets/../../foo.mp4` collapses past projectDir via path.join — this
+    // case used to silently escape; the resolver now strips embedded `..`
+    // segments and re-anchors at the project root.
+    const projectDir = join(tmp, "project");
+    expect(resolveProjectRelativeSrc("assets/../../assets/foo.mp4", projectDir)).toBe(
+      join(projectDir, "assets/foo.mp4"),
+    );
+  });
+
+  it("returns the (non-existent) base-dir path on miss so callers get a stable error message", () => {
+    const projectDir = join(tmp, "project");
+    expect(resolveProjectRelativeSrc("../assets/missing.mp4", projectDir)).toBe(
+      join(projectDir, "../assets/missing.mp4"),
+    );
+  });
+
+  it("prefers compiled-dir over base-dir when the file exists in both", () => {
+    const projectDir = join(tmp, "project");
+    const compiledDir = join(tmp, "compiled");
+    mkdirSync(join(compiledDir, "assets"), { recursive: true });
+    writeFileSync(join(compiledDir, "assets", "foo.mp4"), "");
+    expect(resolveProjectRelativeSrc("assets/foo.mp4", projectDir, compiledDir)).toBe(
+      join(compiledDir, "assets/foo.mp4"),
+    );
+  });
+});
+
 describe("parseVideoElements", () => {
   it("parses videos without an id or data-start attribute", () => {
     const videos = parseVideoElements('<video src="clip.mp4"></video>');

package/src/services/videoFrameExtractor.ts

@@ -7,7 +7,7 @@
 
 import { spawn } from "child_process";
 import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
-import { isAbsolute, join } from "path";
+import { isAbsolute, join, posix, resolve, sep } from "path";
 import { parseHTML } from "linkedom";
 import { extractMediaMetadata, type VideoMetadata } from "../utils/ffprobe.js";
 import {
@@ -230,8 +230,17 @@ export async function extractVideoFramesRange(
   if (isHdr && isMacOS) {
     args.push("-hwaccel", "videotoolbox");
   }
-  if (metadata.hasAlpha && metadata.videoCodec === "vp9") {
-    args.push("-c:v", "libvpx-vp9");
+  // Always force the alpha-aware decoder on codecs that can carry alpha. The
+  // alternative — gating on `metadata.hasAlpha` — relies on tag detection that
+  // has at least three known failure modes: case-sensitivity across ffmpeg
+  // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
+  // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
+  // silently strips alpha during decode and the bug doesn't surface until
+  // the rendered video is missing layers. Codec-based default has no such
+  // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
+  // normally when it isn't.
+  if (codecMayHaveAlpha(metadata.videoCodec)) {
+    args.push("-c:v", decoderForCodec(metadata.videoCodec));
   }
   args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
 
@@ -398,9 +407,31 @@ function resolveSegmentDuration(
   return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
 }
 
+/**
+ * Codecs whose bitstream is allowed to carry an alpha channel. Default the
+ * extraction path to PNG output for these regardless of `metadata.hasAlpha`
+ * so a missed sidecar tag doesn't silently strip transparency. Opaque content
+ * encoded in one of these codecs pays a small file-size cost on the cached
+ * frames but stays correct on the rare case where alpha IS present and the
+ * tag was missed.
+ */
+const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);
+
+export function codecMayHaveAlpha(codec: string | undefined): boolean {
+  return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
+}
+
+export function decoderForCodec(codec: string | undefined): string {
+  const c = (codec ?? "").toLowerCase();
+  if (c === "vp9") return "libvpx-vp9";
+  if (c === "vp8") return "libvpx";
+  return c;
+}
+
 function resolveFrameFormat(metadata: VideoMetadata, requested?: "jpg" | "png"): CacheFrameFormat {
   if (requested) return requested;
-  return metadata.hasAlpha ? "png" : "jpg";
+  if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
+  return "jpg";
 }
 
 /**
@@ -459,6 +490,54 @@ async function convertVfrToCfr(
   }
 }
 
+/**
+ * Resolve a relative `<video src>` to a filesystem path the way the browser
+ * resolves it as a URL. Browsers clamp `..` segments at the served origin's
+ * root; `path.join(projectDir, "../assets/foo")` does not. So a sub-comp
+ * `<video src="../assets/foo">` loads in the page (browser clamps to
+ * `<projectDir>/assets/foo`) but the filesystem-side resolver lands at
+ * `<parentOfProjectDir>/assets/foo` — file missing, extraction skipped,
+ * the rendered output shows the video's first frame for the whole clip.
+ *
+ * The clamp covers two escape patterns: leading `..` (`../assets/foo`) AND
+ * mid-path escapes (`assets/../../foo`) that `path.join` collapses past the
+ * project root silently. Both fall back to a project-rooted candidate that
+ * strips traversal from the resolved path.
+ *
+ * Returns the first existing candidate, or the base-dir join on miss so
+ * the caller's `existsSync` check produces a stable error path.
+ */
+export function resolveProjectRelativeSrc(
+  src: string,
+  baseDir: string,
+  compiledDir?: string,
+): string {
+  const fromCompiled = compiledDir ? join(compiledDir, src) : null;
+  const fromBase = join(baseDir, src);
+  const candidates: string[] = [];
+  if (fromCompiled) candidates.push(fromCompiled);
+  candidates.push(fromBase);
+  // If the joined result escapes the project root (either via leading `..`
+  // or mid-path traversal that path.join collapsed past baseDir), retry
+  // with the basename re-anchored at the project root. This mirrors the
+  // browser URL clamp without relying on a particular `..` shape.
+  const baseAbs = resolve(baseDir);
+  const fromBaseAbs = resolve(fromBase);
+  if (!fromBaseAbs.startsWith(baseAbs + sep) && fromBaseAbs !== baseAbs) {
+    // Normalize first (`assets/../../assets/foo.mp4` → `../assets/foo.mp4`)
+    // then strip any remaining leading `..` segments. Stripping `..` from the
+    // raw input would leave dangling siblings (`assets/../../assets/foo`
+    // would become `assets/assets/foo` instead of `assets/foo`).
+    const normalized = posix.normalize(src.replace(/\\/g, "/"));
+    const stripped = normalized.replace(/^(\.\.\/)+/, "");
+    if (stripped && stripped !== src && !stripped.startsWith("..")) {
+      if (compiledDir) candidates.push(join(compiledDir, stripped));
+      candidates.push(join(baseDir, stripped));
+    }
+  }
+  return candidates.find(existsSync) ?? fromBase;
+}
+
 export async function extractAllVideoFrames(
   videos: VideoElement[],
   baseDir: string,
@@ -487,6 +566,9 @@ export async function extractAllVideoFrames(
   // Phase 1: Resolve paths and download remote videos
   const phase1Start = Date.now();
   const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
+  // Dedupe missing-src warnings: a composition with N <video> elements all
+  // pointing at the same broken src should only print one warning, not N.
+  const warnedSrcs = new Set<string>();
   for (const video of videos) {
     if (signal?.aborted) break;
     try {
@@ -496,9 +578,7 @@ export async function extractAllVideoFrames(
       // baseDir and produce duplicated, nonexistent paths
       // (e.g. C:\tmp\hf-vfr-test-X\C:\tmp\hf-vfr-test-X\vfr_screen.mp4).
       if (!isAbsolute(videoPath) && !isHttpUrl(videoPath)) {
-        const fromCompiled = compiledDir ? join(compiledDir, videoPath) : null;
-        videoPath =
-          fromCompiled && existsSync(fromCompiled) ? fromCompiled : join(baseDir, videoPath);
+        videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
       }
 
       if (isHttpUrl(videoPath)) {
@@ -508,6 +588,19 @@ export async function extractAllVideoFrames(
       }
 
       if (!existsSync(videoPath)) {
+        // Loud: silent miss leaves the rendered video frozen at frame 0 with
+        // no error in stdout — extremely confusing for authors. Dedupe by
+        // src so 50 broken videos pointing at the same path don't spam.
+        if (!warnedSrcs.has(video.src)) {
+          warnedSrcs.add(video.src);
+          process.stderr.write(
+            `[hyperframes:render] WARNING: video src="${video.src}" ` +
+              `could not be resolved on disk (looked for ${videoPath}). ` +
+              `The rendered output will show this video's first frame for the entire clip duration. ` +
+              `If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
+              `(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
+          );
+        }
         errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` });
         continue;
       }

package/src/utils/ffprobe.test.ts

@@ -225,6 +225,46 @@ describe("ffprobe missing-binary fallback", () => {
     expect(meta.hasAlpha).toBe(true);
   });
 
+  // Regression: newer libavformat builds (and the output of `hyperframes
+  // remove-background` itself) write the VP9-alpha sidecar tag as
+  // `ALPHA_MODE` (uppercase). The lowercase-only check classified those
+  // files as having no alpha, the producer extracted them as JPGs, and
+  // the injected <img> overlays were fully opaque rectangles that hid
+  // every static element below them on the z-stack. The bug was silent —
+  // studio preview rendered correctly via native <video> playback while
+  // production renders covered headlines and captions with the avatar.
+  it("extractMediaMetadata detects ALPHA_MODE (uppercase) streams from newer ffmpeg builds", async () => {
+    const { spawn } = createSpawnSpy([
+      {
+        kind: "exit",
+        code: 0,
+        stdout: JSON.stringify({
+          streams: [
+            {
+              codec_type: "video",
+              codec_name: "vp9",
+              width: 320,
+              height: 180,
+              r_frame_rate: "30/1",
+              avg_frame_rate: "30/1",
+              pix_fmt: "yuv420p",
+              tags: { ALPHA_MODE: "1" },
+            },
+          ],
+          format: { duration: "1.5" },
+        }),
+      },
+    ]);
+    vi.resetModules();
+    vi.doMock("child_process", () => ({ spawn }));
+
+    const { extractMediaMetadata: extractMediaMetadataMocked } = await import("./ffprobe.js");
+    const meta = await extractMediaMetadataMocked("/tmp/alpha-uppercase.webm");
+
+    expect(meta.videoCodec).toBe("vp9");
+    expect(meta.hasAlpha).toBe(true);
+  });
+
   it("extractMediaMetadata rethrows ffprobe-missing error for non-image files without fallback", async () => {
     const { spawn } = createSpawnSpy([{ kind: "missing" }]);
     vi.resetModules();

package/src/utils/ffprobe.ts

@@ -203,6 +203,21 @@ function extractStillImageMetadata(filePath: string): StillImageMetadata | null
   }
 }
 
+/**
+ * Read an ffprobe tag case-insensitively. ffmpeg/libavformat versions disagree
+ * on tag casing — VP9 alpha is `alpha_mode` in older builds and `ALPHA_MODE`
+ * in newer ones; HDR tags vary similarly. Use this for any sidecar tag where
+ * you want to be resilient across muxer versions.
+ */
+function readTagCI(tags: Record<string, string | undefined> | undefined, name: string): string {
+  if (!tags) return "";
+  const target = name.toLowerCase();
+  for (const [key, value] of Object.entries(tags)) {
+    if (key.toLowerCase() === target && typeof value === "string") return value;
+  }
+  return "";
+}
+
 function parseFrameRate(frameRateStr: string | undefined): number {
   if (!frameRateStr) return 0;
   const parts = frameRateStr.split("/");
@@ -277,7 +292,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
         : null;
     const colorSpace = ffprobeColorSpace ?? stillImageMeta?.colorSpace ?? null;
     const pixelFormat = videoStream.pix_fmt || "";
-    const alphaMode = videoStream.tags?.alpha_mode || "";
+    const alphaMode = readTagCI(videoStream.tags, "alpha_mode");
     const hasAlpha =
       /(^|[^a-z])yuva|rgba|argb|bgra|gbrap|gray[a-z0-9]*a/i.test(pixelFormat) || alphaMode === "1";