@hyperframes/engine 0.1.0

package/src/services/audioMixer.ts

@@ -0,0 +1,392 @@
+/**
+ * Audio Mixer Service
+ *
+ * Processes and mixes audio tracks using FFmpeg.
+ */
+
+import { existsSync, mkdirSync, rmSync } from "fs";
+import { join, dirname } from "path";
+import { parseHTML } from "linkedom";
+import { extractAudioMetadata } from "../utils/ffprobe.js";
+import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
+import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
+import { runFfmpeg } from "../utils/runFfmpeg.js";
+import type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
+
+export type { AudioElement, AudioTrack, MixResult } from "./audioMixer.types.js";
+
+interface ExtractResult {
+  success: boolean;
+  outputPath: string;
+  durationMs: number;
+  error?: string;
+}
+
+export function parseAudioElements(html: string): AudioElement[] {
+  const elements: AudioElement[] = [];
+  const { document } = parseHTML(html);
+
+  // Parse <audio> elements
+  const audioEls = document.querySelectorAll("audio[id][src]");
+  for (const el of audioEls) {
+    const id = el.getAttribute("id");
+    const src = el.getAttribute("src");
+    if (!id || !src) continue;
+
+    const startAttr = el.getAttribute("data-start");
+    const endAttr = el.getAttribute("data-end");
+    const mediaStartAttr = el.getAttribute("data-media-start");
+    const layerAttr = el.getAttribute("data-layer");
+    const volumeAttr = el.getAttribute("data-volume");
+
+    elements.push({
+      id,
+      src,
+      start: startAttr ? parseFloat(startAttr) : 0,
+      end: endAttr ? parseFloat(endAttr) : 0,
+      mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
+      layer: layerAttr ? parseInt(layerAttr) : 0,
+      volume: volumeAttr ? parseFloat(volumeAttr) : 1.0,
+      type: "audio",
+    });
+  }
+
+  // Parse <video> elements with data-has-audio="true"
+  const videoEls = document.querySelectorAll('video[id][src][data-has-audio="true"]');
+  for (const el of videoEls) {
+    const id = el.getAttribute("id");
+    const src = el.getAttribute("src");
+    if (!id || !src) continue;
+
+    const startAttr = el.getAttribute("data-start");
+    const endAttr = el.getAttribute("data-end");
+    const mediaStartAttr = el.getAttribute("data-media-start");
+    const layerAttr = el.getAttribute("data-layer");
+    const volumeAttr = el.getAttribute("data-volume");
+
+    elements.push({
+      id: `${id}-audio`,
+      src,
+      start: startAttr ? parseFloat(startAttr) : 0,
+      end: endAttr ? parseFloat(endAttr) : 0,
+      mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
+      layer: layerAttr ? parseInt(layerAttr) : 0,
+      volume: volumeAttr ? parseFloat(volumeAttr) : 1.0,
+      type: "video",
+    });
+  }
+
+  return elements;
+}
+
+async function extractAudioFromVideo(
+  videoPath: string,
+  outputPath: string,
+  options?: { startTime?: number; duration?: number },
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<ExtractResult> {
+  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const outputDir = dirname(outputPath);
+  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+
+  const args: string[] = ["-i", videoPath];
+  if (options?.startTime !== undefined) args.push("-ss", String(options.startTime));
+  if (options?.duration !== undefined) args.push("-t", String(options.duration));
+  args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", "-ac", "2", "-y", outputPath);
+
+  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+
+  if (signal?.aborted) {
+    return { success: false, outputPath, durationMs: result.durationMs, error: "Audio extract cancelled" };
+  }
+  if (!result.success) {
+    return {
+      success: false,
+      outputPath,
+      durationMs: result.durationMs,
+      error: result.exitCode !== null ? `FFmpeg exited with code ${result.exitCode}` : result.stderr,
+    };
+  }
+  return { success: true, outputPath, durationMs: result.durationMs };
+}
+
+async function prepareAudioTrack(
+  srcPath: string,
+  outputPath: string,
+  mediaStart: number,
+  duration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<ExtractResult> {
+  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const outputDir = dirname(outputPath);
+  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+
+  const args = [
+    "-ss",
+    String(mediaStart),
+    "-t",
+    String(duration),
+    "-i",
+    srcPath,
+    "-acodec",
+    "pcm_s16le",
+    "-ar",
+    "48000",
+    "-ac",
+    "2",
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+
+  if (signal?.aborted) {
+    return { success: false, outputPath, durationMs: result.durationMs, error: "Audio prepare cancelled" };
+  }
+  return {
+    success: result.success,
+    outputPath,
+    durationMs: result.durationMs,
+    error: !result.success
+      ? result.exitCode !== null
+        ? `FFmpeg exited with code ${result.exitCode}: ${result.stderr.slice(-200)}`
+        : result.stderr
+      : undefined,
+  };
+}
+
+async function generateSilence(
+  outputPath: string,
+  duration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<ExtractResult> {
+  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const outputDir = dirname(outputPath);
+  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+
+  const args = [
+    "-f",
+    "lavfi",
+    "-i",
+    "anullsrc=r=48000:cl=stereo",
+    "-t",
+    String(duration),
+    "-acodec",
+    "pcm_s16le",
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+
+  if (signal?.aborted) {
+    return { success: false, outputPath, durationMs: result.durationMs, error: "Silence generation cancelled" };
+  }
+  return {
+    success: result.success,
+    outputPath,
+    durationMs: result.durationMs,
+    error: !result.success
+      ? result.exitCode !== null
+        ? `FFmpeg exited with code ${result.exitCode}`
+        : result.stderr
+      : undefined,
+  };
+}
+
+async function mixAudioTracks(
+  tracks: AudioTrack[],
+  outputPath: string,
+  totalDuration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
+): Promise<MixResult> {
+  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const masterOutputGain = config?.audioGain ?? DEFAULT_CONFIG.audioGain;
+
+  if (tracks.length === 0) {
+    const result = await generateSilence(outputPath, totalDuration, signal, config);
+    return {
+      success: result.success,
+      outputPath,
+      durationMs: result.durationMs,
+      tracksProcessed: 0,
+      error: result.error,
+    };
+  }
+
+  const outputDir = dirname(outputPath);
+  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+
+  const inputs: string[] = [];
+  const filterParts: string[] = [];
+
+  tracks.forEach((track, i) => {
+    inputs.push("-i", track.srcPath);
+    const delayMs = Math.round(track.start * 1000);
+    const trimDuration = track.end - track.start;
+    filterParts.push(
+      `[${i}:a]atrim=0:${trimDuration},volume=${track.volume},adelay=${delayMs}|${delayMs},apad=whole_dur=${totalDuration}[a${i}]`,
+    );
+  });
+
+  const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
+  const weights = tracks.map(() => "1").join(" ");
+  const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0:normalize=0:weights='${weights}'[mixed]`;
+  const postMixGainFilter = `[mixed]volume=${masterOutputGain}[out]`;
+  const fullFilter = [...filterParts, mixFilter, postMixGainFilter].join(";");
+
+  const args = [
+    ...inputs,
+    "-filter_complex",
+    fullFilter,
+    "-map",
+    "[out]",
+    "-acodec",
+    "aac",
+    "-b:a",
+    "192k",
+    "-t",
+    String(totalDuration),
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
+
+  if (signal?.aborted) {
+    return {
+      success: false,
+      outputPath,
+      durationMs: result.durationMs,
+      tracksProcessed: 0,
+      error: "Audio mix cancelled",
+    };
+  }
+  if (!result.success) {
+    return {
+      success: false,
+      outputPath,
+      durationMs: result.durationMs,
+      tracksProcessed: 0,
+      error: result.exitCode !== null ? `FFmpeg exited with code ${result.exitCode}` : result.stderr,
+    };
+  }
+  return { success: true, outputPath, durationMs: result.durationMs, tracksProcessed: tracks.length };
+}
+
+export async function processCompositionAudio(
+  elements: AudioElement[],
+  baseDir: string,
+  workDir: string,
+  outputPath: string,
+  totalDuration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
+): Promise<MixResult> {
+  const startMs = Date.now();
+  const tracks: AudioTrack[] = [];
+  const errors: string[] = [];
+
+  if (!existsSync(workDir)) mkdirSync(workDir, { recursive: true });
+
+  await Promise.all(
+    elements.map(async (element) => {
+      if (signal?.aborted) {
+        errors.push(`Cancelled: ${element.id}`);
+        return;
+      }
+      try {
+        let srcPath = element.src;
+        if (!srcPath.startsWith("/") && !isHttpUrl(srcPath)) {
+          srcPath = join(baseDir, srcPath);
+        }
+
+        if (isHttpUrl(srcPath)) {
+          try {
+            srcPath = await downloadToTemp(srcPath, workDir);
+          } catch (err: unknown) {
+            errors.push(`Download failed: ${element.id} — ${err instanceof Error ? err.message : String(err)}`);
+            return;
+          }
+        }
+
+        if (!existsSync(srcPath)) {
+          errors.push(`Source not found: ${element.id}`);
+          return;
+        }
+
+        // Fallback: if no duration was specified, probe the actual file
+        if (element.end - element.start <= 0) {
+          const metadata = await extractAudioMetadata(srcPath);
+          const effectiveDuration = metadata.durationSeconds - element.mediaStart;
+          element.end = element.start + (effectiveDuration > 0 ? effectiveDuration : metadata.durationSeconds);
+        }
+
+        let audioSrcPath = srcPath;
+        if (element.type === "video") {
+          const extractedPath = join(workDir, `${element.id}-extracted.wav`);
+          const extractResult = await extractAudioFromVideo(
+            srcPath,
+            extractedPath,
+            {
+              startTime: element.mediaStart,
+              duration: element.end - element.start,
+            },
+            signal,
+            config,
+          );
+          if (!extractResult.success) {
+            errors.push(`Extract failed: ${element.id}`);
+            return;
+          }
+          audioSrcPath = extractedPath;
+        } else {
+          const trimmedPath = join(workDir, `${element.id}-trimmed.wav`);
+          const prepResult = await prepareAudioTrack(
+            srcPath,
+            trimmedPath,
+            element.mediaStart,
+            element.end - element.start,
+            signal,
+            config,
+          );
+          if (!prepResult.success) {
+            errors.push(`Prepare failed: ${element.id}`);
+            return;
+          }
+          audioSrcPath = trimmedPath;
+        }
+
+        tracks.push({
+          id: element.id,
+          srcPath: audioSrcPath,
+          start: element.start,
+          end: element.end,
+          mediaStart: element.mediaStart,
+          duration: element.end - element.start,
+          volume: element.volume || 1.0,
+        });
+      } catch (err: unknown) {
+        errors.push(`Error: ${element.id} — ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }),
+  );
+
+  const mixResult = await mixAudioTracks(tracks, outputPath, totalDuration, signal, config);
+
+  try {
+    rmSync(workDir, { recursive: true, force: true });
+  } catch {
+    /* ignore */
+  }
+
+  return {
+    ...mixResult,
+    durationMs: Date.now() - startMs,
+    error: errors.length > 0 ? `Warnings: ${errors.join(", ")}` : mixResult.error,
+  };
+}

package/src/services/audioMixer.types.ts

@@ -0,0 +1,28 @@
+export interface AudioElement {
+  id: string;
+  src: string;
+  start: number;
+  end: number;
+  mediaStart: number;
+  layer: number;
+  volume?: number;
+  type: "audio" | "video";
+}
+
+export interface AudioTrack {
+  id: string;
+  srcPath: string;
+  start: number;
+  end: number;
+  mediaStart: number;
+  duration: number;
+  volume: number;
+}
+
+export interface MixResult {
+  success: boolean;
+  outputPath: string;
+  durationMs: number;
+  tracksProcessed: number;
+  error?: string;
+}

package/src/services/browserManager.ts

@@ -0,0 +1,219 @@
+/**
+ * Browser Manager
+ *
+ * Manages Puppeteer browser lifecycle: Chrome executable resolution,
+ * launch args, pooled browser acquisition/release.
+ */
+
+import type { Browser, PuppeteerNode } from "puppeteer-core";
+import { existsSync, readdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
+
+let _puppeteer: PuppeteerNode | undefined;
+
+async function getPuppeteer(): Promise<PuppeteerNode> {
+  if (_puppeteer) return _puppeteer;
+  try {
+    const mod = await import("puppeteer" as string);
+    _puppeteer = mod.default;
+  } catch {
+    const mod = await import("puppeteer-core");
+    _puppeteer = mod.default;
+  }
+  if (!_puppeteer) throw new Error("Neither puppeteer nor puppeteer-core found");
+  return _puppeteer;
+}
+
+// "beginframe" = atomic compositor control via HeadlessExperimental.beginFrame (Linux only)
+// "screenshot" = renderSeek + Page.captureScreenshot (all platforms)
+export type CaptureMode = "beginframe" | "screenshot";
+
+export interface AcquiredBrowser {
+  browser: Browser;
+  captureMode: CaptureMode;
+}
+
+/**
+ * Resolve chrome-headless-shell binary for deterministic BeginFrame rendering.
+ * Checks config.chromePath, then PRODUCER_HEADLESS_SHELL_PATH env var,
+ * then scans Puppeteer's managed cache at ~/.cache/puppeteer/chrome-headless-shell/.
+ */
+export function resolveHeadlessShellPath(config?: Partial<Pick<EngineConfig, "chromePath">>): string | undefined {
+  if (config?.chromePath) {
+    return config.chromePath;
+  }
+  if (process.env.PRODUCER_HEADLESS_SHELL_PATH) {
+    return process.env.PRODUCER_HEADLESS_SHELL_PATH;
+  }
+  const baseDir = join(homedir(), ".cache", "puppeteer", "chrome-headless-shell");
+  if (!existsSync(baseDir)) return undefined;
+  try {
+    const versions = readdirSync(baseDir).sort().reverse(); // newest first
+    for (const version of versions) {
+      const candidates = [
+        join(baseDir, version, "chrome-headless-shell-linux64", "chrome-headless-shell"),
+        join(baseDir, version, "chrome-headless-shell-mac-arm64", "chrome-headless-shell"),
+        join(baseDir, version, "chrome-headless-shell-mac-x64", "chrome-headless-shell"),
+        join(baseDir, version, "chrome-headless-shell-win64", "chrome-headless-shell.exe"),
+      ];
+      for (const binary of candidates) {
+        if (existsSync(binary)) return binary;
+      }
+    }
+  } catch {
+    // ignore
+  }
+  return undefined;
+}
+
+let pooledBrowser: Browser | null = null;
+let pooledBrowserRefCount = 0;
+let pooledCaptureMode: CaptureMode = "screenshot";
+
+// Preserve the producer-era export so re-export shims keep the same public API.
+export const ENABLE_BROWSER_POOL = DEFAULT_CONFIG.enableBrowserPool;
+
+export async function acquireBrowser(
+  chromeArgs: string[],
+  config?: Partial<
+    Pick<EngineConfig, "browserTimeout" | "protocolTimeout" | "enableBrowserPool" | "chromePath" | "forceScreenshot">
+  >,
+): Promise<AcquiredBrowser> {
+  const enablePool = config?.enableBrowserPool ?? DEFAULT_CONFIG.enableBrowserPool;
+
+  if (enablePool && pooledBrowser) {
+    pooledBrowserRefCount += 1;
+    return { browser: pooledBrowser, captureMode: pooledCaptureMode };
+  }
+
+  // Config chromePath overrides env var / auto-detection.
+  const headlessShell = resolveHeadlessShellPath(config);
+
+  // BeginFrame requires chrome-headless-shell AND Linux (crashes on macOS/Windows).
+  const isLinux = process.platform === "linux";
+  const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot;
+  let captureMode: CaptureMode;
+  let executablePath: string | undefined;
+
+  if (headlessShell && isLinux && !forceScreenshot) {
+    captureMode = "beginframe";
+    executablePath = headlessShell;
+  } else {
+    // Screenshot mode with renderSeek: works on all platforms.
+    captureMode = "screenshot";
+    if (headlessShell && !isLinux) {
+      executablePath = headlessShell;
+    } else {
+      executablePath = undefined; // Puppeteer uses its bundled Chromium
+    }
+  }
+
+  const ppt = await getPuppeteer();
+  const browser = await ppt.launch({
+    headless: true,
+    args: chromeArgs,
+    defaultViewport: null,
+    executablePath,
+    timeout: config?.browserTimeout ?? DEFAULT_CONFIG.browserTimeout,
+    protocolTimeout: config?.protocolTimeout ?? DEFAULT_CONFIG.protocolTimeout,
+  });
+  if (enablePool) {
+    pooledBrowser = browser;
+    pooledBrowserRefCount = 1;
+    pooledCaptureMode = captureMode;
+  }
+  return { browser, captureMode };
+}
+
+export async function releaseBrowser(
+  browser: Browser,
+  config?: Partial<Pick<EngineConfig, "enableBrowserPool">>,
+): Promise<void> {
+  const enablePool = config?.enableBrowserPool ?? DEFAULT_CONFIG.enableBrowserPool;
+  if (!enablePool) {
+    await browser.close().catch(() => {});
+    return;
+  }
+  if (pooledBrowser && pooledBrowser === browser) {
+    pooledBrowserRefCount = Math.max(0, pooledBrowserRefCount - 1);
+    if (pooledBrowserRefCount === 0) {
+      await browser.close().catch(() => {});
+      pooledBrowser = null;
+    }
+    return;
+  }
+  await browser.close().catch(() => {});
+}
+
+export interface BuildChromeArgsOptions {
+  width: number;
+  height: number;
+  captureMode?: CaptureMode;
+}
+
+export function buildChromeArgs(
+  options: BuildChromeArgsOptions,
+  config?: Partial<Pick<EngineConfig, "disableGpu" | "chromePath">>,
+): string[] {
+  // Chrome flags tuned for headless rendering performance.
+  // Based on Remotion's open-browser.ts flags with additions for our use case.
+  const chromeArgs = [
+    "--no-sandbox",
+    "--disable-setuid-sandbox",
+    "--disable-dev-shm-usage",
+    "--enable-webgl",
+    "--ignore-gpu-blocklist",
+    "--use-gl=angle",
+    "--use-angle=swiftshader",
+    "--font-render-hinting=none",
+    "--force-color-profile=srgb",
+    `--window-size=${options.width},${options.height}`,
+    // Remotion perf flags — prevent Chrome from throttling background tabs/timers
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-renderer-backgrounding",
+    "--disable-background-media-suspend",
+    // Reduce overhead from unused Chrome features
+    "--disable-breakpad",
+    "--disable-component-extensions-with-background-pages",
+    "--disable-default-apps",
+    "--disable-extensions",
+    "--disable-hang-monitor",
+    "--disable-ipc-flooding-protection",
+    "--disable-popup-blocking",
+    "--disable-sync",
+    "--disable-component-update",
+    "--disable-domain-reliability",
+    "--disable-print-preview",
+    "--no-pings",
+    "--no-zygote",
+    // Memory
+    "--force-gpu-mem-available-mb=4096",
+    "--disk-cache-size=268435456",
+    // Disable features that add overhead
+    "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,Translate,BackForwardCache,IntensiveWakeUpThrottling",
+  ];
+
+  // BeginFrame flags — only when using chrome-headless-shell on Linux
+  if (options.captureMode !== "screenshot") {
+    chromeArgs.push(
+      "--deterministic-mode",
+      "--enable-begin-frame-control",
+      "--disable-new-content-rendering-timeout",
+      "--run-all-compositor-stages-before-draw",
+      "--disable-threaded-animation",
+      "--disable-threaded-scrolling",
+      "--disable-checker-imaging",
+      "--disable-image-animation-resync",
+      "--enable-surface-synchronization",
+    );
+  }
+
+  const gpuDisabled = config?.disableGpu ?? DEFAULT_CONFIG.disableGpu;
+  if (gpuDisabled) {
+    chromeArgs.push("--disable-gpu");
+  }
+  return chromeArgs;
+}

package/src/services/chunkEncoder.test.ts

@@ -0,0 +1,27 @@
+import { describe, it, expect } from "vitest";
+import { ENCODER_PRESETS } from "./chunkEncoder.js";
+
+describe("ENCODER_PRESETS", () => {
+  it("has draft, standard, and high presets", () => {
+    expect(ENCODER_PRESETS).toHaveProperty("draft");
+    expect(ENCODER_PRESETS).toHaveProperty("standard");
+    expect(ENCODER_PRESETS).toHaveProperty("high");
+  });
+
+  it("draft uses ultrafast preset with high CRF", () => {
+    expect(ENCODER_PRESETS.draft.preset).toBe("ultrafast");
+    expect(ENCODER_PRESETS.draft.quality).toBeGreaterThan(ENCODER_PRESETS.standard.quality);
+    expect(ENCODER_PRESETS.draft.codec).toBe("h264");
+  });
+
+  it("high uses slow preset with low CRF for better quality", () => {
+    expect(ENCODER_PRESETS.high.preset).toBe("slow");
+    expect(ENCODER_PRESETS.high.quality).toBeLessThan(ENCODER_PRESETS.standard.quality);
+    expect(ENCODER_PRESETS.high.codec).toBe("h264");
+  });
+
+  it("standard sits between draft and high in quality", () => {
+    expect(ENCODER_PRESETS.standard.quality).toBeGreaterThan(ENCODER_PRESETS.high.quality);
+    expect(ENCODER_PRESETS.standard.quality).toBeLessThan(ENCODER_PRESETS.draft.quality);
+  });
+});