@hyperframes/engine 0.4.5 → 0.4.7

package/src/services/frameCapture.ts

@@ -163,6 +163,21 @@ export function isFontResourceError(type: string, text: string, locationUrl: str
   );
 }
 
+async function pollPageExpression(
+  page: Page,
+  expression: string,
+  timeoutMs: number,
+  intervalMs: number = 100,
+): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const ready = Boolean(await page.evaluate(expression));
+    if (ready) return true;
+    await new Promise((resolve) => setTimeout(resolve, intervalMs));
+  }
+  return Boolean(await page.evaluate(expression));
+}
+
 export async function initializeSession(session: CaptureSession): Promise<void> {
   const { page, serverUrl } = session;
 
@@ -213,17 +228,29 @@ export async function initializeSession(session: CaptureSession): Promise<void>
 
     const pageReadyTimeout =
       session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
-    await page.waitForFunction(
+    const pageReady = await pollPageExpression(
+      page,
       `!!(window.__hf && typeof window.__hf.seek === "function" && window.__hf.duration > 0)`,
-      { timeout: pageReadyTimeout },
+      pageReadyTimeout,
     );
+    if (!pageReady) {
+      throw new Error(
+        `[FrameCapture] window.__hf not ready after ${pageReadyTimeout}ms. Page must expose window.__hf = { duration, seek }.`,
+      );
+    }
 
     // Wait for all video elements to have loaded metadata (dimensions + duration)
     // Without this, frame 0 captures videos at their 300x150 default size
-    await page.waitForFunction(
+    const videosReady = await pollPageExpression(
+      page,
       `document.querySelectorAll("video").length === 0 || Array.from(document.querySelectorAll("video")).every(v => v.readyState >= 1)`,
-      { timeout: pageReadyTimeout },
+      pageReadyTimeout,
     );
+    if (!videosReady) {
+      throw new Error(
+        `[FrameCapture] video metadata not ready after ${pageReadyTimeout}ms. Video elements must load metadata before capture starts.`,
+      );
+    }
 
     await page.evaluate(`document.fonts?.ready`);
 

package/src/services/hdrCapture.test.ts

@@ -0,0 +1,159 @@
+import { describe, it, expect } from "vitest";
+import { float16ToPqRgb } from "./hdrCapture.js";
+
+// IEEE 754 half-precision (float16) bit patterns used to feed
+// `float16ToPqRgb`. Encoding rule: sign(1) | exp(5) | frac(10).
+const F16_ZERO = 0x0000; // +0.0
+const F16_HALF = 0x3800; // +0.5  (exp=14, frac=0 → 2^-1)
+const F16_ONE = 0x3c00; // +1.0  (exp=15, frac=0 → 2^0  — SDR white)
+// PQ caps at 10000 nits and SDR_NITS = 203, so the linear input must exceed
+// ~58x SDR white before linearToPQ(L) clips at 1.0. 1024 is well above that.
+const F16_OVERBRIGHT = 0x6400; // +1024.0  (exp=25, frac=0 → 2^10)
+
+function makeFloat16Frame(
+  width: number,
+  height: number,
+  pixel: { r: number; g: number; b: number; a: number },
+  bytesPerRow: number = width * 8,
+): Buffer {
+  // Row-padded layout matches WebGPU readback: bytesPerRow ≥ width * 8 (4
+  // channels × 2 bytes), with garbage bytes after each row's pixel data.
+  const buf = Buffer.alloc(height * bytesPerRow);
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const idx = y * bytesPerRow + x * 8;
+      buf.writeUInt16LE(pixel.r, idx);
+      buf.writeUInt16LE(pixel.g, idx + 2);
+      buf.writeUInt16LE(pixel.b, idx + 4);
+      buf.writeUInt16LE(pixel.a, idx + 6);
+    }
+  }
+  return buf;
+}
+
+describe("float16ToPqRgb", () => {
+  it("returns a buffer of width * height * 6 bytes (rgb48le)", () => {
+    const frame = makeFloat16Frame(4, 3, { r: 0, g: 0, b: 0, a: 0 });
+    const out = float16ToPqRgb(frame, 32, 4, 3);
+    expect(out.length).toBe(4 * 3 * 6);
+  });
+
+  it("encodes float16 black to PQ zero (linearToPQ(0) ≈ 0 after uint16 quantization)", () => {
+    const frame = makeFloat16Frame(2, 2, {
+      r: F16_ZERO,
+      g: F16_ZERO,
+      b: F16_ZERO,
+      a: F16_ZERO,
+    });
+    const out = float16ToPqRgb(frame, 16, 2, 2);
+    for (let i = 0; i < out.length; i += 2) {
+      expect(out.readUInt16LE(i)).toBe(0);
+    }
+  });
+
+  it("clamps overbright float16 input to PQ 65535 (linearToPQ(>>1.0) → 1.0)", () => {
+    // ~1024 linear is well past the 58x-SDR PQ saturation point; output caps
+    // at 1.0 → 65535 in uint16.
+    const frame = makeFloat16Frame(2, 2, {
+      r: F16_OVERBRIGHT,
+      g: F16_OVERBRIGHT,
+      b: F16_OVERBRIGHT,
+      a: F16_ZERO,
+    });
+    const out = float16ToPqRgb(frame, 16, 2, 2);
+    for (let pixel = 0; pixel < 4; pixel++) {
+      const dst = pixel * 6;
+      expect(out.readUInt16LE(dst)).toBe(65535);
+      expect(out.readUInt16LE(dst + 2)).toBe(65535);
+      expect(out.readUInt16LE(dst + 4)).toBe(65535);
+    }
+  });
+
+  it("preserves channel ordering R, G, B (alpha is discarded)", () => {
+    // Distinct float16 values per channel verify the function doesn't
+    // mix them up. Alpha is set high but should not appear in the output.
+    const frame = makeFloat16Frame(1, 1, {
+      r: F16_ONE,
+      g: F16_HALF,
+      b: F16_ZERO,
+      a: F16_ONE,
+    });
+    const out = float16ToPqRgb(frame, 8, 1, 1);
+    const r = out.readUInt16LE(0);
+    const g = out.readUInt16LE(2);
+    const b = out.readUInt16LE(4);
+    expect(r).toBeGreaterThan(g);
+    expect(g).toBeGreaterThan(b);
+    expect(b).toBe(0);
+  });
+
+  it("is monotonic: higher float16 input produces higher PQ output", () => {
+    const dark = makeFloat16Frame(1, 1, { r: F16_ZERO, g: 0, b: 0, a: 0 });
+    const mid = makeFloat16Frame(1, 1, { r: F16_HALF, g: 0, b: 0, a: 0 });
+    const bright = makeFloat16Frame(1, 1, { r: F16_ONE, g: 0, b: 0, a: 0 });
+    const r0 = float16ToPqRgb(dark, 8, 1, 1).readUInt16LE(0);
+    const r1 = float16ToPqRgb(mid, 8, 1, 1).readUInt16LE(0);
+    const r2 = float16ToPqRgb(bright, 8, 1, 1).readUInt16LE(0);
+    expect(r0).toBe(0);
+    expect(r1).toBeGreaterThan(r0);
+    expect(r2).toBeGreaterThan(r1);
+  });
+
+  it("is deterministic across calls with the same input", () => {
+    const frame = makeFloat16Frame(3, 2, {
+      r: F16_HALF,
+      g: F16_ONE,
+      b: F16_ZERO,
+      a: F16_ONE,
+    });
+    const a = float16ToPqRgb(frame, 24, 3, 2);
+    const b = float16ToPqRgb(frame, 24, 3, 2);
+    expect(a.equals(b)).toBe(true);
+  });
+
+  it("handles padded bytesPerRow (WebGPU 256-byte alignment)", () => {
+    // WebGPU readback pads rows to 256-byte multiples. For a 4-pixel-wide
+    // frame the actual pixel data is 32 bytes but bytesPerRow is 256.
+    const width = 4;
+    const height = 2;
+    const bytesPerRow = 256;
+    const frame = makeFloat16Frame(
+      width,
+      height,
+      { r: F16_HALF, g: F16_HALF, b: F16_HALF, a: 0 },
+      bytesPerRow,
+    );
+    const out = float16ToPqRgb(frame, bytesPerRow, width, height);
+    expect(out.length).toBe(width * height * 6);
+    // Every R component should be the same non-zero value (uniform input).
+    const expected = out.readUInt16LE(0);
+    expect(expected).toBeGreaterThan(0);
+    for (let pixel = 0; pixel < width * height; pixel++) {
+      expect(out.readUInt16LE(pixel * 6)).toBe(expected);
+    }
+  });
+
+  it("ignores garbage bytes in the row padding region", () => {
+    // Stuff junk into the trailing padding to make sure the PQ encoder
+    // walks via bytesPerRow stride and not via raw buffer position.
+    const width = 2;
+    const height = 2;
+    const bytesPerRow = 64;
+    const frame = makeFloat16Frame(
+      width,
+      height,
+      { r: F16_ZERO, g: F16_ZERO, b: F16_ZERO, a: F16_ZERO },
+      bytesPerRow,
+    );
+    for (let y = 0; y < height; y++) {
+      const padStart = y * bytesPerRow + width * 8;
+      for (let i = padStart; i < (y + 1) * bytesPerRow; i++) {
+        frame[i] = 0xff;
+      }
+    }
+    const out = float16ToPqRgb(frame, bytesPerRow, width, height);
+    for (let i = 0; i < out.length; i += 2) {
+      expect(out.readUInt16LE(i)).toBe(0);
+    }
+  });
+});

package/src/services/hdrCapture.ts

@@ -0,0 +1,354 @@
+/// <reference types="@webgpu/types" />
+/**
+ * HDR Capture Service
+ *
+ * Captures HDR video frames via WebGPU float16 readback.
+ *
+ * The pipeline:
+ * 1. FFmpeg extracts raw HDR pixels (rgba64le) from video sources
+ * 2. Node converts HLG/PQ signal → linear light → float16
+ * 3. writeTexture uploads float16 data to WebGPU rgba16float texture
+ * 4. (Optional) WebGPU shader applies GSAP CSS transform
+ * 5. readback extracts float16 RGBA via base64 transfer
+ * 6. Node converts linear float16 → PQ signal → pipe to FFmpeg H.265
+ *
+ * Requirements:
+ * - Headed Chrome (not headless) — WebGPU unavailable in headless mode
+ * - GPU access (Metal on macOS, Vulkan+NVIDIA on Linux)
+ *
+ * Performance: ~6 fps at 1080x1920 via base64 transfer.
+ */
+
+import type { Page, Browser, PuppeteerNode } from "puppeteer-core";
+import { existsSync, readdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+
+// ── PQ (SMPTE 2084) OETF ─────────────────────────────────────────────────────
+
+const PQ_M1 = 0.1593017578125;
+const PQ_M2 = 78.84375;
+const PQ_C1 = 0.8359375;
+const PQ_C2 = 18.8515625;
+const PQ_C3 = 18.6875;
+const PQ_MAX_NITS = 10000.0;
+const SDR_NITS = 203.0;
+
+function linearToPQ(L: number): number {
+  const Lp = Math.max(0, (L * SDR_NITS) / PQ_MAX_NITS);
+  const Lm1 = Math.pow(Lp, PQ_M1);
+  return Math.pow((PQ_C1 + PQ_C2 * Lm1) / (1.0 + PQ_C3 * Lm1), PQ_M2);
+}
+
+function float16Decode(h: number): number {
+  const sign = (h >> 15) & 1;
+  const exp = (h >> 10) & 0x1f;
+  const frac = h & 0x3ff;
+  if (exp === 0) return (sign ? -1 : 1) * Math.pow(2, -14) * (frac / 1024);
+  if (exp === 31) return frac ? NaN : sign ? -Infinity : Infinity;
+  return (sign ? -1 : 1) * Math.pow(2, exp - 15) * (1 + frac / 1024);
+}
+
+// ── Browser-side interface ────────────────────────────────────────────────────
+
+interface HdrCaptureRuntime {
+  uploadAndReadback(float16Base64: string): Promise<{ base64: string; bytesPerRow: number }>;
+}
+
+// ── Initialization ────────────────────────────────────────────────────────────
+
+/**
+ * Inject the WebGPU HDR readback runtime into the page.
+ *
+ * Creates an rgba16float render texture that accepts writeTexture uploads
+ * and provides readback via base64 transfer.
+ */
+export async function initHdrReadback(page: Page, width: number, height: number): Promise<boolean> {
+  return page.evaluate(
+    async (w: number, h: number): Promise<boolean> => {
+      if (!navigator.gpu) return false;
+
+      const adapter = await navigator.gpu.requestAdapter();
+      if (!adapter) return false;
+
+      const device = await adapter.requestDevice();
+
+      const bytesPerPixel = 8; // rgba16float = 4 channels × 2 bytes
+      const bytesPerRow = Math.ceil((w * bytesPerPixel) / 256) * 256;
+
+      // Render texture — includes COPY_DST for writeTexture uploads
+      const renderTexture = device.createTexture({
+        size: [w, h],
+        format: "rgba16float",
+        usage:
+          GPUTextureUsage.RENDER_ATTACHMENT |
+          GPUTextureUsage.COPY_SRC |
+          GPUTextureUsage.COPY_DST |
+          GPUTextureUsage.TEXTURE_BINDING,
+      });
+
+      const readBuffer = device.createBuffer({
+        size: bytesPerRow * h,
+        usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
+      });
+
+      const captureRuntime = {
+        device,
+        renderTexture,
+        readBuffer,
+        bytesPerRow,
+        width: w,
+        height: h,
+
+        /**
+         * Upload pre-converted float16 RGBA data and read it back.
+         * The float16 data must be row-aligned to bytesPerRow.
+         *
+         * Input: base64-encoded Uint16Array (float16 RGBA, row-padded)
+         * Output: base64-encoded readback of the same texture
+         */
+        async uploadAndReadback(
+          float16Base64: string,
+        ): Promise<{ base64: string; bytesPerRow: number }> {
+          // Decode base64 → Uint8Array
+          const binary = atob(float16Base64);
+          const bytes = new Uint8Array(binary.length);
+          for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+
+          // Upload to texture
+          device.queue.writeTexture(
+            { texture: renderTexture },
+            bytes.buffer,
+            { bytesPerRow, rowsPerImage: h },
+            [w, h],
+          );
+
+          // Readback
+          const encoder = device.createCommandEncoder();
+          encoder.copyTextureToBuffer(
+            { texture: renderTexture },
+            { buffer: readBuffer, bytesPerRow },
+            [w, h],
+          );
+          device.queue.submit([encoder.finish()]);
+
+          await readBuffer.mapAsync(GPUMapMode.READ);
+          const readBytes = new Uint8Array(readBuffer.getMappedRange().slice(0));
+          readBuffer.unmap();
+
+          // Base64 encode in chunks
+          let b64 = "";
+          const chunkSize = 32768;
+          for (let i = 0; i < readBytes.length; i += chunkSize) {
+            const slice = readBytes.subarray(i, Math.min(i + chunkSize, readBytes.length));
+            b64 += String.fromCharCode(...slice);
+          }
+
+          return { base64: btoa(b64), bytesPerRow };
+        },
+      };
+
+      (window as unknown as Record<string, unknown>).__hfHdrCapture = captureRuntime;
+      return true;
+    },
+    width,
+    height,
+  );
+}
+
+// ── HDR frame conversion ──────────────────────────────────────────────────────
+
+/**
+ * Convert raw rgba64le pixels (from FFmpeg) to a base64 string for FFmpeg encoding.
+ *
+ * For HLG sources: the pixel values are already HLG-encoded. We pass them through
+ * as-is (normalized to 16-bit) and tag the output as HLG. No OETF conversion needed —
+ * the HLG signal values ARE the correct encoding. Converting to linear and back to
+ * PQ produces worse results because every viewer's PQ→display tone-mapping differs
+ * from its HLG→display tone-mapping.
+ *
+ * The WebGPU round-trip is skipped for pass-through — the pixels go directly from
+ * FFmpeg extraction to FFmpeg encoding. WebGPU is only needed when transforms
+ * (scale, rotate, opacity from GSAP) must be applied to the HDR pixels.
+ */
+export function convertHdrFrameToRgb48le(
+  rawRgba64le: Buffer,
+  width: number,
+  height: number,
+): Buffer {
+  const input = new Uint16Array(
+    rawRgba64le.buffer,
+    rawRgba64le.byteOffset,
+    rawRgba64le.byteLength / 2,
+  );
+
+  // Convert RGBA → RGB (drop alpha) for rgb48le output
+  const output = Buffer.alloc(width * height * 6);
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const srcIdx = (y * width + x) * 4;
+      const dstIdx = (y * width + x) * 6;
+      output.writeUInt16LE(input[srcIdx] ?? 0, dstIdx);
+      output.writeUInt16LE(input[srcIdx + 1] ?? 0, dstIdx + 2);
+      output.writeUInt16LE(input[srcIdx + 2] ?? 0, dstIdx + 4);
+    }
+  }
+
+  return output;
+}
+
+// ── Frame upload + readback ───────────────────────────────────────────────────
+
+/**
+ * Upload a float16 frame to WebGPU and read it back.
+ * Call after converting with convertHdrFrameToFloat16Base64.
+ */
+export async function uploadAndReadbackHdrFrame(
+  page: Page,
+  float16Base64: string,
+): Promise<{ rawBuffer: Buffer; bytesPerRow: number }> {
+  const result = await page.evaluate(
+    async (b64: string): Promise<{ base64: string; bytesPerRow: number }> => {
+      const hdr = (window as unknown as Record<string, unknown>).__hfHdrCapture as
+        | HdrCaptureRuntime
+        | undefined;
+      if (!hdr) throw new Error("HDR capture not initialized");
+      return hdr.uploadAndReadback(b64);
+    },
+    float16Base64,
+  );
+
+  return {
+    rawBuffer: Buffer.from(result.base64, "base64"),
+    bytesPerRow: result.bytesPerRow,
+  };
+}
+
+// ── PQ conversion ─────────────────────────────────────────────────────────────
+
+/**
+ * Convert float16 RGBA readback to PQ-encoded rgb48le for FFmpeg.
+ */
+export function float16ToPqRgb(
+  rawBuffer: Buffer,
+  bytesPerRow: number,
+  width: number,
+  height: number,
+): Buffer {
+  const data = new Uint16Array(rawBuffer.buffer, rawBuffer.byteOffset, rawBuffer.byteLength / 2);
+  const channelsPerRow = bytesPerRow / 2;
+  const output = Buffer.alloc(width * height * 6);
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const srcIdx = y * channelsPerRow + x * 4;
+      const r = float16Decode(data[srcIdx] ?? 0);
+      const g = float16Decode(data[srcIdx + 1] ?? 0);
+      const b = float16Decode(data[srcIdx + 2] ?? 0);
+
+      const dstIdx = (y * width + x) * 6;
+      output.writeUInt16LE(Math.round(Math.min(1.0, linearToPQ(r)) * 65535), dstIdx);
+      output.writeUInt16LE(Math.round(Math.min(1.0, linearToPQ(g)) * 65535), dstIdx + 2);
+      output.writeUInt16LE(Math.round(Math.min(1.0, linearToPQ(b)) * 65535), dstIdx + 4);
+    }
+  }
+
+  return output;
+}
+
+// ── Chrome launch ─────────────────────────────────────────────────────────────
+
+function resolveHeadedChromePath(): string | undefined {
+  const baseDir = join(homedir(), ".cache", "puppeteer", "chrome");
+  if (!existsSync(baseDir)) return undefined;
+  const versions = readdirSync(baseDir).sort().reverse();
+  for (const version of versions) {
+    const candidates = [
+      join(
+        baseDir,
+        version,
+        "chrome-mac-arm64",
+        "Google Chrome for Testing.app",
+        "Contents",
+        "MacOS",
+        "Google Chrome for Testing",
+      ),
+      join(
+        baseDir,
+        version,
+        "chrome-mac-x64",
+        "Google Chrome for Testing.app",
+        "Contents",
+        "MacOS",
+        "Google Chrome for Testing",
+      ),
+      join(baseDir, version, "chrome-linux64", "chrome"),
+      join(baseDir, version, "chrome-win64", "chrome.exe"),
+    ];
+    for (const binary of candidates) {
+      if (existsSync(binary)) return binary;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Launch a headed Chrome browser with WebGPU enabled.
+ */
+export async function launchHdrBrowser(
+  width: number,
+  height: number,
+): Promise<{ browser: Browser; page: Page }> {
+  let ppt: PuppeteerNode | undefined;
+  try {
+    const mod = await import("puppeteer" as string);
+    ppt = mod.default;
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException | undefined)?.code;
+    if (code !== "ERR_MODULE_NOT_FOUND" && code !== "MODULE_NOT_FOUND") {
+      throw err;
+    }
+    const mod = await import("puppeteer-core");
+    ppt = mod.default;
+  }
+  if (!ppt) throw new Error("Neither puppeteer nor puppeteer-core found");
+
+  const chromePath = resolveHeadedChromePath();
+  if (!chromePath) {
+    throw new Error(
+      "[HDR] No Chrome binary found. Install: npx @puppeteer/browsers install chrome@stable",
+    );
+  }
+
+  const browser = await ppt.launch({
+    headless: false,
+    executablePath: chromePath,
+    args: buildHdrChromeArgs(width, height),
+  });
+
+  const page = await browser.newPage();
+  await page.setViewport({ width, height });
+
+  return { browser, page };
+}
+
+export function buildHdrChromeArgs(width: number, height: number): string[] {
+  return [
+    "--enable-unsafe-webgpu",
+    "--no-sandbox",
+    "--disable-setuid-sandbox",
+    "--window-position=-10000,-10000",
+    `--window-size=${width},${height}`,
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-renderer-backgrounding",
+    "--disable-background-media-suspend",
+    "--disable-extensions",
+    "--disable-component-update",
+    "--disable-default-apps",
+    "--disable-sync",
+    "--no-zygote",
+    "--force-gpu-mem-available-mb=4096",
+  ];
+}