@hyperframes/engine 0.1.0

package/src/services/streamingEncoder.ts

@@ -0,0 +1,308 @@
+/**
+ * Streaming Encoder Service
+ *
+ * Pipes frame screenshot buffers directly to FFmpeg's stdin instead of writing
+ * them to disk and reading them back in a separate encode stage.  Follows the
+ * Remotion pattern of image2pipe → FFmpeg.
+ *
+ * Two building blocks:
+ *   1. Frame reorder buffer – ensures out-of-order parallel workers feed
+ *      frames to FFmpeg stdin in sequential order.
+ *   2. Streaming FFmpeg encoder – spawns FFmpeg with `-f image2pipe` and
+ *      exposes a `writeFrame(buffer)` + `close()` API.
+ */
+
+import { spawn, type ChildProcess } from "child_process";
+import { existsSync, mkdirSync, statSync } from "fs";
+import { dirname } from "path";
+
+import { type GpuEncoder, getCachedGpuEncoder, getGpuEncoderName } from "../utils/gpuEncoder.js";
+import { type EncoderOptions } from "./chunkEncoder.types.js";
+import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
+
+// Re-export EncoderOptions so callers can reference the type via this module.
+export type { EncoderOptions } from "./chunkEncoder.types.js";
+
+// ---------------------------------------------------------------------------
+// 1. Frame reorder buffer (based on Remotion's ensure-frames-in-order.ts)
+// ---------------------------------------------------------------------------
+
+export interface FrameReorderBuffer {
+  waitForFrame: (frame: number) => Promise<void>;
+  advanceTo: (frame: number) => void;
+  waitForAllDone: () => Promise<void>;
+}
+
+export function createFrameReorderBuffer(startFrame: number, endFrame: number): FrameReorderBuffer {
+  let nextFrame = startFrame;
+  let waiters: Array<{ frame: number; resolve: () => void }> = [];
+
+  const resolveWaiters = () => {
+    for (const waiter of waiters.slice()) {
+      if (waiter.frame === nextFrame) {
+        waiter.resolve();
+        waiters = waiters.filter((w) => w !== waiter);
+      }
+    }
+  };
+
+  return {
+    waitForFrame: (frame: number) =>
+      new Promise<void>((resolve) => {
+        waiters.push({ frame, resolve });
+        resolveWaiters();
+      }),
+    advanceTo: (frame: number) => {
+      nextFrame = frame;
+      resolveWaiters();
+    },
+    waitForAllDone: () =>
+      new Promise<void>((resolve) => {
+        waiters.push({ frame: endFrame, resolve });
+        resolveWaiters();
+      }),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// 2. Streaming FFmpeg encoder
+// ---------------------------------------------------------------------------
+
+export interface StreamingEncoderOptions {
+  fps: number;
+  width: number;
+  height: number;
+  codec?: "h264" | "h265" | "vp9" | "prores";
+  preset?: string;
+  quality?: number;
+  bitrate?: string;
+  pixelFormat?: string;
+  useGpu?: boolean;
+  imageFormat?: "jpeg" | "png";
+}
+
+export interface StreamingEncoderResult {
+  success: boolean;
+  durationMs: number;
+  fileSize: number;
+  error?: string;
+}
+
+export interface StreamingEncoder {
+  writeFrame: (buffer: Buffer) => boolean;
+  close: () => Promise<StreamingEncoderResult>;
+  getExitStatus: () => "running" | "success" | "error";
+}
+
+/**
+ * Build FFmpeg args for streaming (image2pipe) input.
+ * Reuses the same codec/quality/GPU logic as chunkEncoder's buildEncoderArgs
+ * but with `-f image2pipe` instead of `-i <pattern>`.
+ */
+function buildStreamingArgs(
+  options: StreamingEncoderOptions,
+  outputPath: string,
+  gpuEncoder: GpuEncoder = null,
+): string[] {
+  const {
+    fps,
+    codec = "h264",
+    preset = "medium",
+    quality = 23,
+    bitrate,
+    pixelFormat = "yuv420p",
+    useGpu = false,
+    imageFormat = "jpeg",
+  } = options;
+
+  // Input args: pipe from stdin
+  const inputCodec = imageFormat === "png" ? "png" : "mjpeg";
+  const args: string[] = [
+    "-f",
+    "image2pipe",
+    "-vcodec",
+    inputCodec,
+    "-framerate",
+    String(fps),
+    "-i",
+    "-",
+    "-r",
+    String(fps),
+  ];
+
+  const shouldUseGpu = useGpu && gpuEncoder !== null;
+
+  if (codec === "h264" || codec === "h265") {
+    if (shouldUseGpu) {
+      const encoderName = getGpuEncoderName(gpuEncoder, codec);
+      args.push("-c:v", encoderName);
+
+      switch (gpuEncoder) {
+        case "nvenc":
+          args.push("-preset", preset);
+          if (bitrate) args.push("-b:v", bitrate);
+          else args.push("-cq", String(quality));
+          break;
+        case "videotoolbox":
+          if (bitrate) args.push("-b:v", bitrate);
+          else {
+            const vtQuality = Math.max(0, Math.min(100, 100 - quality * 2));
+            args.push("-q:v", String(vtQuality));
+          }
+          args.push("-allow_sw", "1");
+          break;
+        case "vaapi":
+          args.unshift("-vaapi_device", "/dev/dri/renderD128");
+          args.push("-vf", "format=nv12,hwupload");
+          if (bitrate) args.push("-b:v", bitrate);
+          else args.push("-qp", String(quality));
+          break;
+        case "qsv":
+          args.push("-preset", preset);
+          if (bitrate) args.push("-b:v", bitrate);
+          else args.push("-global_quality", String(quality));
+          break;
+      }
+    } else {
+      const encoderName = codec === "h264" ? "libx264" : "libx265";
+      args.push("-c:v", encoderName, "-preset", preset);
+      if (bitrate) args.push("-b:v", bitrate);
+      else args.push("-crf", String(quality));
+    }
+  } else if (codec === "vp9") {
+    args.push("-c:v", "libvpx-vp9", "-b:v", bitrate || "0", "-crf", String(quality));
+    args.push("-deadline", preset === "ultrafast" ? "realtime" : "good");
+  } else if (codec === "prores") {
+    args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
+    return [...args, "-y", outputPath];
+  }
+
+  if (gpuEncoder !== "vaapi") {
+    args.push("-pix_fmt", pixelFormat);
+  }
+
+  args.push("-y", outputPath);
+  return args;
+}
+
+/**
+ * Spawn a streaming FFmpeg encoder that accepts frame buffers on stdin.
+ */
+export async function spawnStreamingEncoder(
+  outputPath: string,
+  options: StreamingEncoderOptions,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegStreamingTimeout">>,
+): Promise<StreamingEncoder> {
+  const outputDir = dirname(outputPath);
+  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+
+  let gpuEncoder: GpuEncoder = null;
+  if (options.useGpu) {
+    gpuEncoder = await getCachedGpuEncoder();
+  }
+
+  const args = buildStreamingArgs(options, outputPath, gpuEncoder);
+
+  const startTime = Date.now();
+  const ffmpeg: ChildProcess = spawn("ffmpeg", args, {
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+
+  let exitStatus: "running" | "success" | "error" = "running";
+  let stderr = "";
+  let exitCode: number | null = null;
+  let exitPromiseResolve: ((value: void) => void) | null = null;
+  const exitPromise = new Promise<void>((resolve) => (exitPromiseResolve = resolve));
+
+  // Track stderr for progress and error messages
+  ffmpeg.stderr?.on("data", (data: Buffer) => {
+    stderr += data.toString();
+  });
+
+  ffmpeg.on("close", (code: number | null) => {
+    exitCode = code;
+    exitStatus = code === 0 ? "success" : "error";
+    exitPromiseResolve?.();
+  });
+
+  ffmpeg.on("error", (err: Error) => {
+    exitStatus = "error";
+    stderr += `\nProcess error: ${err.message}`;
+    exitPromiseResolve?.();
+  });
+
+  // Handle abort signal
+  const onAbort = () => {
+    if (exitStatus === "running") {
+      ffmpeg.kill("SIGTERM");
+    }
+  };
+  if (signal) {
+    if (signal.aborted) {
+      ffmpeg.kill("SIGTERM");
+    } else {
+      signal.addEventListener("abort", onAbort, { once: true });
+    }
+  }
+
+  // Timeout safety
+  const streamingTimeout = config?.ffmpegStreamingTimeout ?? DEFAULT_CONFIG.ffmpegStreamingTimeout;
+  const timer = setTimeout(() => {
+    if (exitStatus === "running") {
+      ffmpeg.kill("SIGTERM");
+    }
+  }, streamingTimeout);
+
+  const encoder: StreamingEncoder = {
+    writeFrame: (buffer: Buffer): boolean => {
+      if (exitStatus !== "running" || !ffmpeg.stdin || ffmpeg.stdin.destroyed) {
+        return false;
+      }
+      return ffmpeg.stdin.write(buffer);
+    },
+
+    close: async (): Promise<StreamingEncoderResult> => {
+      clearTimeout(timer);
+      if (signal) signal.removeEventListener("abort", onAbort);
+
+      // Close stdin to signal end of input
+      if (ffmpeg.stdin && !ffmpeg.stdin.destroyed) {
+        await new Promise<void>((resolve) => {
+          ffmpeg.stdin!.end(() => resolve());
+        });
+      }
+
+      // Wait for FFmpeg to finish
+      await exitPromise;
+
+      const durationMs = Date.now() - startTime;
+
+      if (signal?.aborted) {
+        return {
+          success: false,
+          durationMs,
+          fileSize: 0,
+          error: "Streaming encode cancelled",
+        };
+      }
+
+      if (exitCode !== 0) {
+        return {
+          success: false,
+          durationMs,
+          fileSize: 0,
+          error: `FFmpeg exited with code ${exitCode}`,
+        };
+      }
+
+      const fileSize = existsSync(outputPath) ? statSync(outputPath).size : 0;
+
+      return { success: true, durationMs, fileSize };
+    },
+
+    getExitStatus: () => exitStatus,
+  };
+
+  return encoder;
+}

package/src/services/videoFrameExtractor.ts

@@ -0,0 +1,395 @@
+/**
+ * Video Frame Extractor Service
+ *
+ * Pre-extracts video frames using FFmpeg for frame-accurate rendering.
+ * Videos are replaced with <img> elements during capture.
+ */
+
+import { spawn } from "child_process";
+import { existsSync, mkdirSync, readdirSync, rmSync } from "fs";
+import { join } from "path";
+import { parseHTML } from "linkedom";
+import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js";
+import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js";
+import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
+
+export interface VideoElement {
+  id: string;
+  src: string;
+  start: number;
+  end: number;
+  mediaStart: number;
+  hasAudio: boolean;
+}
+
+export interface ExtractedFrames {
+  videoId: string;
+  srcPath: string;
+  outputDir: string;
+  framePattern: string;
+  fps: number;
+  totalFrames: number;
+  metadata: VideoMetadata;
+  framePaths: Map<number, string>;
+}
+
+export interface ExtractionOptions {
+  fps: number;
+  outputDir: string;
+  quality?: number;
+  format?: "jpg" | "png";
+}
+
+export interface ExtractionResult {
+  success: boolean;
+  extracted: ExtractedFrames[];
+  errors: Array<{ videoId: string; error: string }>;
+  totalFramesExtracted: number;
+  durationMs: number;
+}
+
+export function parseVideoElements(html: string): VideoElement[] {
+  const videos: VideoElement[] = [];
+  const { document } = parseHTML(html);
+
+  const videoEls = document.querySelectorAll("video[id][src]");
+  for (const el of videoEls) {
+    const id = el.getAttribute("id");
+    const src = el.getAttribute("src");
+    if (!id || !src) continue;
+
+    const startAttr = el.getAttribute("data-start");
+    const endAttr = el.getAttribute("data-end");
+    const mediaStartAttr = el.getAttribute("data-media-start");
+    const hasAudioAttr = el.getAttribute("data-has-audio");
+
+    videos.push({
+      id,
+      src,
+      start: startAttr ? parseFloat(startAttr) : 0,
+      end: endAttr ? parseFloat(endAttr) : 0,
+      mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
+      hasAudio: hasAudioAttr === "true",
+    });
+  }
+
+  return videos;
+}
+
+export async function extractVideoFramesRange(
+  videoPath: string,
+  videoId: string,
+  startTime: number,
+  duration: number,
+  options: ExtractionOptions,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<ExtractedFrames> {
+  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+  const { fps, outputDir, quality = 95, format = "jpg" } = options;
+
+  const videoOutputDir = join(outputDir, videoId);
+  if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });
+
+  const metadata = await extractVideoMetadata(videoPath);
+  const framePattern = `frame_%05d.${format}`;
+  const outputPattern = join(videoOutputDir, framePattern);
+
+  const args: string[] = [
+    "-ss",
+    String(startTime),
+    "-i",
+    videoPath,
+    "-t",
+    String(duration),
+    "-vf",
+    `fps=${fps}`,
+    "-q:v",
+    format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0",
+  ];
+  if (format === "png") args.push("-compression_level", "6");
+  args.push("-y", outputPattern);
+
+  return new Promise((resolve, reject) => {
+    const ffmpeg = spawn("ffmpeg", args);
+    let stderr = "";
+    const onAbort = () => {
+      ffmpeg.kill("SIGTERM");
+    };
+    if (signal) {
+      if (signal.aborted) {
+        ffmpeg.kill("SIGTERM");
+      } else {
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+    }
+
+    const timer = setTimeout(() => {
+      ffmpeg.kill("SIGTERM");
+    }, ffmpegProcessTimeout);
+
+    ffmpeg.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    ffmpeg.on("close", (code) => {
+      clearTimeout(timer);
+      if (signal) signal.removeEventListener("abort", onAbort);
+      if (signal?.aborted) {
+        reject(new Error("Video frame extraction cancelled"));
+        return;
+      }
+      if (code !== 0) {
+        reject(new Error(`FFmpeg exited with code ${code}: ${stderr.slice(-500)}`));
+        return;
+      }
+
+      const framePaths = new Map<number, string>();
+      const files = readdirSync(videoOutputDir)
+        .filter((f) => f.startsWith("frame_") && f.endsWith(`.${format}`))
+        .sort();
+      files.forEach((file, index) => {
+        framePaths.set(index, join(videoOutputDir, file));
+      });
+
+      resolve({
+        videoId,
+        srcPath: videoPath,
+        outputDir: videoOutputDir,
+        framePattern,
+        fps,
+        totalFrames: framePaths.size,
+        metadata,
+        framePaths,
+      });
+    });
+
+    ffmpeg.on("error", (err) => {
+      clearTimeout(timer);
+      if (signal) signal.removeEventListener("abort", onAbort);
+      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+        reject(new Error("[FFmpeg] ffmpeg not found"));
+      } else {
+        reject(err);
+      }
+    });
+  });
+}
+
+export async function extractAllVideoFrames(
+  videos: VideoElement[],
+  baseDir: string,
+  options: ExtractionOptions,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<ExtractionResult> {
+  const startTime = Date.now();
+  const extracted: ExtractedFrames[] = [];
+  const errors: Array<{ videoId: string; error: string }> = [];
+  let totalFramesExtracted = 0;
+
+  // Process videos in parallel for better performance
+  const results = await Promise.all(
+    videos.map(async (video) => {
+      if (signal?.aborted) {
+        throw new Error("Video frame extraction cancelled");
+      }
+      try {
+        let videoPath = video.src;
+        if (!videoPath.startsWith("/") && !isHttpUrl(videoPath)) {
+          videoPath = join(baseDir, videoPath);
+        }
+
+        if (isHttpUrl(videoPath)) {
+          const downloadDir = join(options.outputDir, "_downloads");
+          mkdirSync(downloadDir, { recursive: true });
+          videoPath = await downloadToTemp(videoPath, downloadDir);
+        }
+
+        if (!existsSync(videoPath)) {
+          return { error: { videoId: video.id, error: `Video file not found: ${videoPath}` } };
+        }
+
+        let videoDuration = video.end - video.start;
+
+        // Fallback: if no data-duration/data-end was specified, probe the actual file
+        if (videoDuration <= 0) {
+          const metadata = await extractVideoMetadata(videoPath);
+          const sourceDuration = metadata.durationSeconds - video.mediaStart;
+          videoDuration = sourceDuration > 0 ? sourceDuration : metadata.durationSeconds;
+          video.end = video.start + videoDuration;
+        }
+
+        const result = await extractVideoFramesRange(
+          videoPath,
+          video.id,
+          video.mediaStart,
+          videoDuration,
+          options,
+          signal,
+          config,
+        );
+
+        return { result };
+      } catch (err) {
+        return {
+          error: {
+            videoId: video.id,
+            error: err instanceof Error ? err.message : String(err),
+          },
+        };
+      }
+    }),
+  );
+
+  // Collect results and errors
+  for (const item of results) {
+    if ("error" in item && item.error) {
+      errors.push(item.error);
+    } else if ("result" in item) {
+      extracted.push(item.result);
+      totalFramesExtracted += item.result.totalFrames;
+    }
+  }
+
+  return { success: errors.length === 0, extracted, errors, totalFramesExtracted, durationMs: Date.now() - startTime };
+}
+
+export function getFrameAtTime(extracted: ExtractedFrames, globalTime: number, videoStart: number): string | null {
+  const localTime = globalTime - videoStart;
+  if (localTime < 0) return null;
+  const frameIndex = Math.floor(localTime * extracted.fps);
+  if (frameIndex < 0 || frameIndex >= extracted.totalFrames) return null;
+  return extracted.framePaths.get(frameIndex) || null;
+}
+
+export class FrameLookupTable {
+  private videos: Map<
+    string,
+    {
+      extracted: ExtractedFrames;
+      start: number;
+      end: number;
+      mediaStart: number;
+    }
+  > = new Map();
+  private orderedVideos: Array<{
+    videoId: string;
+    extracted: ExtractedFrames;
+    start: number;
+    end: number;
+    mediaStart: number;
+  }> = [];
+  private activeVideoIds: Set<string> = new Set();
+  private startCursor = 0;
+  private lastTime: number | null = null;
+
+  addVideo(extracted: ExtractedFrames, start: number, end: number, mediaStart: number): void {
+    this.videos.set(extracted.videoId, { extracted, start, end, mediaStart });
+    this.orderedVideos = Array.from(this.videos.entries())
+      .map(([videoId, video]) => ({ videoId, ...video }))
+      .sort((a, b) => a.start - b.start);
+    this.resetActiveState();
+  }
+
+  getFrame(videoId: string, globalTime: number): string | null {
+    const video = this.videos.get(videoId);
+    if (!video) return null;
+    if (globalTime < video.start || globalTime >= video.end) return null;
+    return getFrameAtTime(video.extracted, globalTime, video.start);
+  }
+
+  private resetActiveState(): void {
+    this.activeVideoIds.clear();
+    this.startCursor = 0;
+    this.lastTime = null;
+  }
+
+  private refreshActiveSet(globalTime: number): void {
+    if (this.lastTime == null || globalTime < this.lastTime) {
+      this.activeVideoIds.clear();
+      this.startCursor = 0;
+      for (const entry of this.orderedVideos) {
+        if (entry.start <= globalTime && globalTime < entry.end) {
+          this.activeVideoIds.add(entry.videoId);
+        }
+        if (entry.start <= globalTime) {
+          this.startCursor += 1;
+        } else {
+          break;
+        }
+      }
+      this.lastTime = globalTime;
+      return;
+    }
+
+    while (this.startCursor < this.orderedVideos.length) {
+      const candidate = this.orderedVideos[this.startCursor];
+      if (!candidate) break;
+      if (candidate.start > globalTime) {
+        break;
+      }
+      if (globalTime < candidate.end) {
+        this.activeVideoIds.add(candidate.videoId);
+      }
+      this.startCursor += 1;
+    }
+
+    for (const videoId of Array.from(this.activeVideoIds)) {
+      const video = this.videos.get(videoId);
+      if (!video || globalTime < video.start || globalTime >= video.end) {
+        this.activeVideoIds.delete(videoId);
+      }
+    }
+    this.lastTime = globalTime;
+  }
+
+  getActiveFramePayloads(globalTime: number): Map<string, { framePath: string; frameIndex: number }> {
+    const frames = new Map<string, { framePath: string; frameIndex: number }>();
+    this.refreshActiveSet(globalTime);
+    for (const videoId of this.activeVideoIds) {
+      const video = this.videos.get(videoId);
+      if (!video) continue;
+      const localTime = globalTime - video.start;
+      const frameIndex = Math.floor(localTime * video.extracted.fps);
+      if (frameIndex < 0 || frameIndex >= video.extracted.totalFrames) continue;
+      const framePath = video.extracted.framePaths.get(frameIndex);
+      if (!framePath) continue;
+      frames.set(videoId, { framePath, frameIndex });
+    }
+    return frames;
+  }
+
+  getActiveFrames(globalTime: number): Map<string, string> {
+    const payloads = this.getActiveFramePayloads(globalTime);
+    const frames = new Map<string, string>();
+    for (const [videoId, payload] of payloads) {
+      frames.set(videoId, payload.framePath);
+    }
+    return frames;
+  }
+
+  cleanup(): void {
+    for (const video of this.videos.values()) {
+      if (existsSync(video.extracted.outputDir)) {
+        rmSync(video.extracted.outputDir, { recursive: true, force: true });
+      }
+    }
+    this.videos.clear();
+    this.orderedVideos = [];
+    this.resetActiveState();
+  }
+}
+
+export function createFrameLookupTable(videos: VideoElement[], extracted: ExtractedFrames[]): FrameLookupTable {
+  const table = new FrameLookupTable();
+  const extractedMap = new Map<string, ExtractedFrames>();
+  for (const ext of extracted) extractedMap.set(ext.videoId, ext);
+
+  for (const video of videos) {
+    const ext = extractedMap.get(video.id);
+    if (ext) table.addVideo(ext, video.start, video.end, video.mediaStart);
+  }
+
+  return table;
+}