@hyperframes/engine 0.4.6 → 0.4.8

package/src/services/streamingEncoder.test.ts

@@ -0,0 +1,228 @@
+/**
+ * buildStreamingArgs unit tests.
+ *
+ * These tests focus on the FFmpeg CLI shape rather than spawning the encoder
+ * — they're the cheap regression net for the HDR static-metadata bug
+ * (side_data=[none] in the encoded MP4) reproduced by
+ * packages/producer/scripts/hdr-smoke.ts. Without these assertions, future
+ * refactors of the x265-params string can silently strip
+ * master-display / max-cll and ship as SDR BT.2020 again.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import {
+  buildStreamingArgs,
+  createFrameReorderBuffer,
+  type StreamingEncoderOptions,
+} from "./streamingEncoder.js";
+import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js";
+
+const baseHdrPq: StreamingEncoderOptions = {
+  fps: 30,
+  width: 1920,
+  height: 1080,
+  codec: "h265",
+  preset: "medium",
+  quality: 23,
+  pixelFormat: "yuv420p10le",
+  useGpu: false,
+  rawInputFormat: "rgb48le",
+  hdr: { transfer: "pq" },
+};
+
+const baseHdrHlg: StreamingEncoderOptions = {
+  ...baseHdrPq,
+  hdr: { transfer: "hlg" },
+};
+
+const baseSdr: StreamingEncoderOptions = {
+  fps: 30,
+  width: 1920,
+  height: 1080,
+  codec: "h264",
+  preset: "medium",
+  quality: 23,
+  useGpu: false,
+};
+
+function getX265ParamsValue(args: string[]): string | undefined {
+  const idx = args.indexOf("-x265-params");
+  return idx === -1 ? undefined : args[idx + 1];
+}
+
+describe("buildStreamingArgs", () => {
+  describe("HDR PQ (libx265)", () => {
+    it("emits master-display and max-cll in -x265-params", () => {
+      const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
+      const x265 = getX265ParamsValue(args);
+      expect(x265).toBeDefined();
+      expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
+      expect(x265).toContain(`max-cll=${DEFAULT_HDR10_MASTERING.maxCll}`);
+      expect(x265).toContain("colorprim=bt2020");
+      expect(x265).toContain("transfer=smpte2084");
+      expect(x265).toContain("colormatrix=bt2020nc");
+    });
+
+    it("tags the output stream with bt2020 / smpte2084 / tv range", () => {
+      const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
+      expect(args).toContain("-colorspace:v");
+      expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt2020nc");
+      expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt2020");
+      expect(args[args.indexOf("-color_trc:v") + 1]).toBe("smpte2084");
+      expect(args[args.indexOf("-color_range") + 1]).toBe("tv");
+    });
+
+    it("uses libx265 with -tag:v hvc1 for QuickTime compatibility", () => {
+      const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
+      const cvIdx = args.indexOf("-c:v");
+      expect(cvIdx).toBeGreaterThan(-1);
+      expect(args[cvIdx + 1]).toBe("libx265");
+      expect(args).toContain("-tag:v");
+      expect(args[args.indexOf("-tag:v") + 1]).toBe("hvc1");
+    });
+
+    it("keeps the aq-mode prefix even with master-display present", () => {
+      const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
+      const x265 = getX265ParamsValue(args);
+      expect(x265?.startsWith("aq-mode=3")).toBe(true);
+    });
+
+    it("uses the simpler aq-mode prefix on ultrafast preset", () => {
+      const args = buildStreamingArgs({ ...baseHdrPq, preset: "ultrafast" }, "/tmp/out.mp4");
+      const x265 = getX265ParamsValue(args);
+      expect(x265?.startsWith("aq-mode=3:")).toBe(true);
+      expect(x265).not.toContain("aq-strength");
+      expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
+    });
+  });
+
+  describe("HDR HLG (libx265)", () => {
+    it("emits master-display, max-cll, and the HLG transfer", () => {
+      const args = buildStreamingArgs(baseHdrHlg, "/tmp/out.mp4");
+      const x265 = getX265ParamsValue(args);
+      expect(x265).toContain("transfer=arib-std-b67");
+      expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
+      expect(x265).toContain(`max-cll=${DEFAULT_HDR10_MASTERING.maxCll}`);
+    });
+
+    it("tags the output stream with arib-std-b67", () => {
+      const args = buildStreamingArgs(baseHdrHlg, "/tmp/out.mp4");
+      expect(args[args.indexOf("-color_trc:v") + 1]).toBe("arib-std-b67");
+    });
+  });
+
+  describe("HDR raw input tagging", () => {
+    it("tags the rawvideo input with the matching color metadata", () => {
+      const args = buildStreamingArgs(baseHdrPq, "/tmp/out.mp4");
+      const inputColorTrcIdx = args.indexOf("-color_trc");
+      expect(inputColorTrcIdx).toBeGreaterThan(-1);
+      expect(args[inputColorTrcIdx + 1]).toBe("smpte2084");
+      const inputPrimariesIdx = args.indexOf("-color_primaries");
+      expect(inputPrimariesIdx).toBeGreaterThan(-1);
+      expect(args[inputPrimariesIdx + 1]).toBe("bt2020");
+      // Pix_fmt of the raw input must match the buffer we hand FFmpeg.
+      expect(args.indexOf("rgb48le")).toBeGreaterThan(-1);
+    });
+
+    it("does not strip the input color tags when bitrate is set instead of CRF", () => {
+      const args = buildStreamingArgs({ ...baseHdrPq, bitrate: "20M" }, "/tmp/out.mp4");
+      const x265 = getX265ParamsValue(args);
+      expect(x265).toContain(`master-display=${DEFAULT_HDR10_MASTERING.masterDisplay}`);
+      expect(args).toContain("-b:v");
+      expect(args[args.indexOf("-b:v") + 1]).toBe("20M");
+    });
+  });
+
+  describe("SDR fallback", () => {
+    it("does NOT emit HDR mastering metadata for SDR encodes", () => {
+      const args = buildStreamingArgs(baseSdr, "/tmp/out.mp4");
+      const x264 = args[args.indexOf("-x264-params") + 1];
+      expect(x264).toContain("colorprim=bt709");
+      expect(x264).toContain("transfer=bt709");
+      expect(x264).toContain("colormatrix=bt709");
+      expect(x264).not.toContain("master-display");
+      expect(x264).not.toContain("max-cll");
+    });
+
+    it("tags SDR output with bt709 and tv range", () => {
+      const args = buildStreamingArgs(baseSdr, "/tmp/out.mp4");
+      expect(args[args.indexOf("-color_trc:v") + 1]).toBe("bt709");
+      expect(args[args.indexOf("-color_primaries:v") + 1]).toBe("bt709");
+      expect(args[args.indexOf("-colorspace:v") + 1]).toBe("bt709");
+      expect(args[args.indexOf("-color_range") + 1]).toBe("tv");
+    });
+  });
+
+  describe("output path", () => {
+    it("places the output path last after -y", () => {
+      const args = buildStreamingArgs(baseHdrPq, "/tmp/some-output.mp4");
+      expect(args[args.length - 2]).toBe("-y");
+      expect(args[args.length - 1]).toBe("/tmp/some-output.mp4");
+    });
+  });
+});
+
+describe("createFrameReorderBuffer", () => {
+  it("fast-paths waitForFrame(cursor) without queueing", async () => {
+    const buf = createFrameReorderBuffer(0, 3);
+    await buf.waitForFrame(0);
+  });
+
+  it("gates out-of-order writers into cursor order", async () => {
+    const buf = createFrameReorderBuffer(0, 4);
+    const writeOrder: number[] = [];
+
+    const writer = async (frame: number) => {
+      await buf.waitForFrame(frame);
+      writeOrder.push(frame);
+      buf.advanceTo(frame + 1);
+    };
+
+    const p3 = writer(3);
+    const p1 = writer(1);
+    const p2 = writer(2);
+    const p0 = writer(0);
+
+    await Promise.all([p0, p1, p2, p3]);
+    expect(writeOrder).toEqual([0, 1, 2, 3]);
+  });
+
+  it("supports multiple waiters registered for the same frame", async () => {
+    const buf = createFrameReorderBuffer(0, 2);
+    const resolved: string[] = [];
+
+    const a = buf.waitForFrame(1).then(() => resolved.push("a"));
+    const b = buf.waitForFrame(1).then(() => resolved.push("b"));
+
+    buf.advanceTo(0);
+    await Promise.resolve();
+    expect(resolved).toEqual([]);
+
+    buf.advanceTo(1);
+    await Promise.all([a, b]);
+    expect(resolved.sort()).toEqual(["a", "b"]);
+  });
+
+  it("waitForAllDone resolves when cursor reaches endFrame", async () => {
+    const buf = createFrameReorderBuffer(0, 3);
+    let done = false;
+    const allDone = buf.waitForAllDone().then(() => {
+      done = true;
+    });
+
+    buf.advanceTo(1);
+    await Promise.resolve();
+    expect(done).toBe(false);
+
+    buf.advanceTo(3);
+    await allDone;
+    expect(done).toBe(true);
+  });
+
+  it("waitForAllDone fast-paths when cursor already past endFrame", async () => {
+    const buf = createFrameReorderBuffer(0, 3);
+    buf.advanceTo(5);
+    await buf.waitForAllDone();
+  });
+});

package/src/services/streamingEncoder.ts

@@ -1,9 +1,9 @@
 /**
  * Streaming Encoder Service
  *
- * Pipes frame screenshot buffers directly to FFmpeg's stdin instead of writing
- * them to disk and reading them back in a separate encode stage.  Follows the
- * Remotion pattern of image2pipe → FFmpeg.
+ * Pipes frame screenshot buffers directly to FFmpeg's stdin via `-f image2pipe`
+ * instead of writing them to disk and reading them back in a separate encode
+ * stage. Inspired by Remotion's approach to browser-based video rendering.
  *
  * Two building blocks:
  *   1. Frame reorder buffer – ensures out-of-order parallel workers feed
@@ -17,6 +17,7 @@ import { existsSync, mkdirSync, statSync } from "fs";
 import { dirname } from "path";
 
 import { type GpuEncoder, getCachedGpuEncoder, getGpuEncoderName } from "../utils/gpuEncoder.js";
+import { getHdrEncoderColorParams } from "../utils/hdr.js";
 import { type EncoderOptions } from "./chunkEncoder.types.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 
@@ -24,8 +25,16 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 export type { EncoderOptions } from "./chunkEncoder.types.js";
 
 // ---------------------------------------------------------------------------
-// 1. Frame reorder buffer (based on Remotion's ensure-frames-in-order.ts)
+// 1. Frame reorder buffer — ordered async barrier
 // ---------------------------------------------------------------------------
+//
+// Parallel workers produce frames out of order; FFmpeg's stdin expects them in
+// strict sequential order. Each worker calls `waitForFrame(n)` to block until
+// its turn, writes, then calls `advanceTo(n + 1)` to release the next waiter.
+//
+// `pending` holds an array per frame index (not a single resolver) so that
+// `waitForAllDone` can coexist with the writer still waiting on the final
+// frame without one clobbering the other.
 
 export interface FrameReorderBuffer {
   waitForFrame: (frame: number) => Promise<void>;
@@ -34,34 +43,49 @@ export interface FrameReorderBuffer {
 }
 
 export function createFrameReorderBuffer(startFrame: number, endFrame: number): FrameReorderBuffer {
-  let nextFrame = startFrame;
-  let waiters: Array<{ frame: number; resolve: () => void }> = [];
-
-  const resolveWaiters = () => {
-    for (const waiter of waiters.slice()) {
-      if (waiter.frame === nextFrame) {
-        waiter.resolve();
-        waiters = waiters.filter((w) => w !== waiter);
-      }
+  let cursor = startFrame;
+  const pending = new Map<number, Array<() => void>>();
+
+  const enqueueAt = (frame: number, resolve: () => void): void => {
+    const list = pending.get(frame);
+    if (list === undefined) {
+      pending.set(frame, [resolve]);
+    } else {
+      list.push(resolve);
     }
   };
 
-  return {
-    waitForFrame: (frame: number) =>
-      new Promise<void>((resolve) => {
-        waiters.push({ frame, resolve });
-        resolveWaiters();
-      }),
-    advanceTo: (frame: number) => {
-      nextFrame = frame;
-      resolveWaiters();
-    },
-    waitForAllDone: () =>
-      new Promise<void>((resolve) => {
-        waiters.push({ frame: endFrame, resolve });
-        resolveWaiters();
-      }),
+  const flushAt = (frame: number): void => {
+    const list = pending.get(frame);
+    if (list === undefined) return;
+    pending.delete(frame);
+    for (const resolve of list) resolve();
   };
+
+  const waitForFrame = (frame: number): Promise<void> =>
+    new Promise<void>((resolve) => {
+      if (frame === cursor) {
+        resolve();
+        return;
+      }
+      enqueueAt(frame, resolve);
+    });
+
+  const advanceTo = (frame: number): void => {
+    cursor = frame;
+    flushAt(frame);
+  };
+
+  const waitForAllDone = (): Promise<void> =>
+    new Promise<void>((resolve) => {
+      if (cursor >= endFrame) {
+        resolve();
+        return;
+      }
+      enqueueAt(endFrame, resolve);
+    });
+
+  return { waitForFrame, advanceTo, waitForAllDone };
 }
 
 // ---------------------------------------------------------------------------
@@ -79,6 +103,9 @@ export interface StreamingEncoderOptions {
   pixelFormat?: string;
   useGpu?: boolean;
   imageFormat?: "jpeg" | "png";
+  hdr?: { transfer: import("../utils/hdr.js").HdrTransfer };
+  /** When set, use rawvideo input instead of image2pipe. For HDR PQ-encoded frames. */
+  rawInputFormat?: "rgb48le";
 }
 
 export interface StreamingEncoderResult {
@@ -98,8 +125,11 @@ export interface StreamingEncoder {
  * Build FFmpeg args for streaming (image2pipe) input.
  * Reuses the same codec/quality/GPU logic as chunkEncoder's buildEncoderArgs
  * but with `-f image2pipe` instead of `-i <pattern>`.
+ *
+ * Exported so unit tests can assert on the constructed CLI without spawning
+ * FFmpeg — see streamingEncoder.test.ts.
  */
-function buildStreamingArgs(
+export function buildStreamingArgs(
   options: StreamingEncoderOptions,
   outputPath: string,
   gpuEncoder: GpuEncoder = null,
@@ -116,19 +146,41 @@ function buildStreamingArgs(
   } = options;
 
   // Input args: pipe from stdin
-  const inputCodec = imageFormat === "png" ? "png" : "mjpeg";
-  const args: string[] = [
-    "-f",
-    "image2pipe",
-    "-vcodec",
-    inputCodec,
-    "-framerate",
-    String(fps),
-    "-i",
-    "-",
-    "-r",
-    String(fps),
-  ];
+  const args: string[] = [];
+  if (options.rawInputFormat) {
+    // Raw pixel input (HLG/PQ-encoded rgb48le from FFmpeg extraction).
+    // Tag the input with the correct color space so FFmpeg uses the right
+    // YUV matrix when converting rgb48le → yuv420p10le for encoding.
+    // Without these tags FFmpeg assumes bt709 and applies the wrong matrix.
+    const hdrTransfer = options.hdr?.transfer;
+    const inputColorTrc =
+      hdrTransfer === "pq" ? "smpte2084" : hdrTransfer === "hlg" ? "arib-std-b67" : undefined;
+    args.push(
+      "-f",
+      "rawvideo",
+      "-pix_fmt",
+      options.rawInputFormat,
+      "-s",
+      `${options.width}x${options.height}`,
+      "-framerate",
+      String(fps),
+    );
+    if (inputColorTrc) {
+      args.push(
+        "-color_primaries",
+        "bt2020",
+        "-color_trc",
+        inputColorTrc,
+        "-colorspace",
+        "bt2020nc",
+      );
+    }
+    args.push("-i", "-");
+  } else {
+    const inputCodec = imageFormat === "png" ? "png" : "mjpeg";
+    args.push("-f", "image2pipe", "-vcodec", inputCodec, "-framerate", String(fps), "-i", "-");
+  }
+  args.push("-r", String(fps));
 
   const shouldUseGpu = useGpu && gpuEncoder !== null;
 
@@ -169,16 +221,25 @@ function buildStreamingArgs(
       if (bitrate) args.push("-b:v", bitrate);
       else args.push("-crf", String(quality));
 
-      // Encoder-specific params: anti-banding + bt709 color space.
-      // aq-mode=3 redistributes bits to dark flat areas (gradients).
-      // colorprim/transfer/colormatrix embed bt709 in the H.264/H.265 VUI.
+      // Encoder-specific params: anti-banding + color space tagging.
+      // For HDR, getHdrEncoderColorParams also emits the SMPTE ST 2086
+      // mastering-display and CTA-861.3 MaxCLL/MaxFALL SEI messages —
+      // without them, players (Apple, YouTube, HDR TVs) treat the file
+      // as SDR BT.2020 and tone-map incorrectly.
       const xParamsFlag = codec === "h264" ? "-x264-params" : "-x265-params";
-      const colorParams = "colorprim=bt709:transfer=bt709:colormatrix=bt709";
+      const colorParams =
+        options.rawInputFormat && options.hdr
+          ? getHdrEncoderColorParams(options.hdr.transfer).x265ColorParams
+          : "colorprim=bt709:transfer=bt709:colormatrix=bt709";
       if (preset === "ultrafast") {
         args.push(xParamsFlag, `aq-mode=3:${colorParams}`);
       } else {
         args.push(xParamsFlag, `aq-mode=3:aq-strength=0.8:deblock=1,1:${colorParams}`);
       }
+      // Apple devices require hvc1 tag for HEVC playback (default hev1 won't open in QuickTime)
+      if (codec === "h265") {
+        args.push("-tag:v", "hvc1");
+      }
     }
   } else if (codec === "vp9") {
     args.push("-c:v", "libvpx-vp9", "-b:v", bitrate || "0", "-crf", String(quality));
@@ -194,27 +255,47 @@ function buildStreamingArgs(
     return [...args, "-y", outputPath];
   }
 
-  // BT.709 color space metadata — Chrome screenshots are sRGB which maps to bt709.
-  // Tags the output so players interpret colors correctly across devices.
+  // Color space metadata.
+  // When rawInputFormat is set, data comes from the WebGPU HDR pipeline
+  // (PQ-encoded) — tag with bt2020/PQ truthfully.
+  // Otherwise, Chrome captures sRGB — tag as bt709.
   if (codec === "h264" || codec === "h265") {
-    args.push(
-      "-colorspace:v",
-      "bt709",
-      "-color_primaries:v",
-      "bt709",
-      "-color_trc:v",
-      "bt709",
-      "-color_range",
-      "tv",
-    );
+    if (options.rawInputFormat && options.hdr) {
+      args.push(
+        "-colorspace:v",
+        "bt2020nc",
+        "-color_primaries:v",
+        "bt2020",
+        "-color_trc:v",
+        options.hdr.transfer === "pq" ? "smpte2084" : "arib-std-b67",
+        "-color_range",
+        "tv",
+      );
+    } else {
+      args.push(
+        "-colorspace:v",
+        "bt709",
+        "-color_primaries:v",
+        "bt709",
+        "-color_trc:v",
+        "bt709",
+        "-color_range",
+        "tv",
+      );
+    }
 
-    // Convert full-range RGB input (Chrome screenshots) to limited/TV range for H.264.
-    if (gpuEncoder === "vaapi") {
+    // Video filter for range/color conversion.
+    // Raw HDR input (from WebGPU pipeline) is already PQ-encoded — no conversion needed.
+    // Chrome screenshots need full→TV range conversion.
+    if (options.rawInputFormat) {
+      // No filter needed — PQ data goes straight to encoder
+    } else if (gpuEncoder === "vaapi") {
       const vfIdx = args.indexOf("-vf");
       if (vfIdx !== -1) {
         args[vfIdx + 1] = `scale=in_range=pc:out_range=tv,${args[vfIdx + 1]}`;
       }
     } else if (!shouldUseGpu) {
+      // Range conversion: Chrome screenshots are full-range RGB.
       args.push("-vf", "scale=in_range=pc:out_range=tv");
     }
 
@@ -304,7 +385,15 @@ export async function spawnStreamingEncoder(
       if (exitStatus !== "running" || !ffmpeg.stdin || ffmpeg.stdin.destroyed) {
         return false;
       }
-      return ffmpeg.stdin.write(buffer);
+      // Copy the buffer before writing — Node streams hold a reference to the
+      // provided buffer and drain it asynchronously. The HDR path's compositor
+      // reuses pre-allocated transOutput/normalCanvas buffers across frames,
+      // so without this copy the pipe would read partially-overwritten data
+      // and flicker. The SDR path doesn't invoke writeFrame at all (it pipes
+      // PNG files via encodeFramesFromDir), so the memcpy here is HDR-only
+      // and justified by correctness.
+      const copy = Buffer.from(buffer);
+      return ffmpeg.stdin.write(copy);
     },
 
     close: async (): Promise<StreamingEncoderResult> => {
@@ -312,9 +401,10 @@ export async function spawnStreamingEncoder(
       if (signal) signal.removeEventListener("abort", onAbort);
 
       // Close stdin to signal end of input
-      if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
+      const stdin = ffmpeg.stdin;
+      if (stdin && !stdin.destroyed) {
         await new Promise<void>((resolve) => {
-          ffmpeg.stdin!.end(() => resolve());
+          stdin.end(() => resolve());
         });
       }