@bendyline/squisq-video-react 1.0.0

package/src/hooks/useFrameCapture.ts

@@ -0,0 +1,263 @@
+/**
+ * useFrameCapture — Hidden div + html2canvas frame capture.
+ *
+ * Mounts a DocPlayer in renderMode inside a hidden div (same document),
+ * then captures individual frames by seeking the player and rendering
+ * the DOM to a canvas via html2canvas.
+ *
+ * Uses React directly — no script injection, no iframes, no eval.
+ *
+ * Returns an ImageBitmap for each frame (transferable to a Worker).
+ */
+
+import { createElement } from 'react';
+import { createRoot, type Root } from 'react-dom/client';
+import { useRef, useCallback, useMemo } from 'react';
+import type { Doc, MediaProvider } from '@bendyline/squisq/schemas';
+import type { RenderHtmlOptions } from '@bendyline/squisq-video';
+import { DocPlayer, MediaContext } from '@bendyline/squisq-react';
+import type { SquisqWindow, CaptionMode, CaptionStyle } from '@bendyline/squisq-react';
+import html2canvas from 'html2canvas';
+
+export interface FrameCaptureHandle {
+  /** Initialize the hidden player. Returns the video duration in seconds. */
+  init: (
+    doc: Doc,
+    renderOptions: Omit<RenderHtmlOptions, 'playerScript'>,
+    captionMode?: CaptionMode,
+  ) => Promise<number>;
+  /** Capture a single frame at the given time (seconds). Returns an ImageBitmap. */
+  captureFrame: (time: number) => Promise<ImageBitmap>;
+  /** Clean up resources. */
+  destroy: () => void;
+}
+
+/** Extension → MIME type map (hoisted to avoid per-image allocation). */
+const MIME_MAP: Record<string, string> = {
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  png: 'image/png',
+  gif: 'image/gif',
+  webp: 'image/webp',
+  svg: 'image/svg+xml',
+  bmp: 'image/bmp',
+  avif: 'image/avif',
+};
+
+/** Convert an ArrayBuffer to a base64 data URI using chunked encoding (O(n)). */
+function arrayBufferToDataUrl(buffer: ArrayBuffer, mime: string): string {
+  const bytes = new Uint8Array(buffer);
+  const chunks: string[] = [];
+  const CHUNK = 8192;
+  for (let i = 0; i < bytes.length; i += CHUNK) {
+    chunks.push(String.fromCharCode(...bytes.subarray(i, i + CHUNK)));
+  }
+  return `data:${mime};base64,${btoa(chunks.join(''))}`;
+}
+
+/**
+ * Create an inline MediaProvider from a map of paths to ArrayBuffers.
+ */
+function createInlineProvider(images: Map<string, ArrayBuffer>): MediaProvider {
+  const dataUrls = new Map<string, string>();
+  const mimeTypes = new Map<string, string>();
+  for (const [path, buffer] of images) {
+    const ext = path.split('.').pop()?.toLowerCase() ?? '';
+    const mime = MIME_MAP[ext] ?? 'application/octet-stream';
+    dataUrls.set(path, arrayBufferToDataUrl(buffer, mime));
+    mimeTypes.set(path, mime);
+  }
+
+  return {
+    async resolveUrl(relativePath: string): Promise<string> {
+      return dataUrls.get(relativePath) ?? relativePath;
+    },
+    async listMedia() {
+      return [...dataUrls.keys()].map((name) => ({
+        name,
+        mimeType: mimeTypes.get(name) ?? 'application/octet-stream',
+        size: 0,
+      }));
+    },
+    async addMedia() {
+      throw new Error('Read-only');
+    },
+    async removeMedia() {
+      throw new Error('Read-only');
+    },
+    dispose() {},
+  };
+}
+
+/**
+ * Hook that manages a hidden div for frame capture.
+ */
+export function useFrameCapture(): FrameCaptureHandle {
+  const containerRef = useRef<HTMLDivElement | null>(null);
+  const rootRef = useRef<Root | null>(null);
+  const dimensionsRef = useRef<{ width: number; height: number }>({ width: 1920, height: 1080 });
+
+  const init = useCallback(
+    async (
+      doc: Doc,
+      renderOptions: Omit<RenderHtmlOptions, 'playerScript'>,
+      captionMode?: CaptionMode,
+    ): Promise<number> => {
+      // Clean up any existing container.
+      // Defer unmount to avoid "synchronously unmount a root while React
+      // was already rendering" when init() is called from a React handler.
+      if (rootRef.current || containerRef.current) {
+        const oldRoot = rootRef.current;
+        const oldContainer = containerRef.current;
+        rootRef.current = null;
+        containerRef.current = null;
+        await new Promise<void>((resolve) => {
+          setTimeout(() => {
+            if (oldRoot) oldRoot.unmount();
+            if (oldContainer) oldContainer.remove();
+            resolve();
+          }, 0);
+        });
+      }
+
+      const width = renderOptions.width ?? 1920;
+      const height = renderOptions.height ?? 1080;
+      dimensionsRef.current = { width, height };
+
+      // Create a hidden container
+      const container = document.createElement('div');
+      container.style.cssText =
+        `position:fixed;left:0;top:0;width:${width}px;height:${height}px;` +
+        'opacity:0;pointer-events:none;z-index:-1;overflow:hidden;';
+      document.body.appendChild(container);
+      containerRef.current = container;
+
+      // Create render root
+      const renderRoot = document.createElement('div');
+      renderRoot.id = 'squisq-capture-root';
+      renderRoot.style.cssText = `width:${width}px;height:${height}px;`;
+      container.appendChild(renderRoot);
+
+      // Build media provider from images
+      const mediaProvider = renderOptions.images
+        ? createInlineProvider(renderOptions.images)
+        : null;
+
+      // Mount DocPlayer in renderMode via React
+      const root = createRoot(renderRoot);
+      rootRef.current = root;
+
+      // Derive caption props from captionMode
+      const captionsEnabled = captionMode !== undefined && captionMode !== 'off';
+      const captionStyle: CaptionStyle = captionMode === 'social' ? 'social' : 'standard';
+
+      const playerElement = createElement(DocPlayer, {
+        script: doc,
+        basePath: '.',
+        renderMode: true,
+        showControls: false,
+        autoPlay: false,
+        forceViewport: { width, height, name: 'export' },
+        captionsEnabled,
+        captionStyle,
+      });
+
+      // Defer rendering to the next microtask to avoid "synchronously unmount
+      // a root while React was already rendering" when init() is called during
+      // a React render cycle (e.g., from startExport in VideoExportModal).
+      await new Promise<void>((resolve) => setTimeout(resolve, 0));
+
+      if (mediaProvider) {
+        root.render(createElement(MediaContext.Provider, { value: mediaProvider }, playerElement));
+      } else {
+        root.render(playerElement);
+      }
+
+      // Wait for the render API to appear on window
+      return new Promise<number>((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          const w = window as SquisqWindow;
+          const hasSeek = typeof w.seekTo === 'function';
+          const hasDur = typeof w.getDuration === 'function';
+          const rootEl = containerRef.current?.querySelector('#squisq-capture-root');
+          const hasPlayer = rootEl ? rootEl.querySelector('.doc-player') !== null : false;
+          reject(
+            new Error(
+              `Render API did not initialize within 15s. ` +
+                `seekTo=${hasSeek}, getDuration=${hasDur}, player=${hasPlayer}, root=${!!rootEl}`,
+            ),
+          );
+        }, 15000);
+
+        const checkApi = () => {
+          const w = window as SquisqWindow;
+          if (typeof w.getDuration === 'function' && typeof w.seekTo === 'function') {
+            clearTimeout(timeout);
+            const duration = w.getDuration();
+            resolve(duration);
+          } else {
+            requestAnimationFrame(checkApi);
+          }
+        };
+
+        // Give React time to mount and run useEffects
+        setTimeout(checkApi, 500);
+      });
+    },
+    [],
+  );
+
+  const captureFrame = useCallback(async (time: number): Promise<ImageBitmap> => {
+    const container = containerRef.current;
+    const w = window as SquisqWindow;
+    if (!container || typeof w.seekTo !== 'function') {
+      throw new Error('Frame capture not initialized — call init() first');
+    }
+
+    const { width, height } = dimensionsRef.current;
+
+    // Seek the player to the target time
+    await w.seekTo(time);
+
+    // Wait for the DOM to update after seek
+    await new Promise<void>((resolve) =>
+      requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
+    );
+
+    const root = container.querySelector('#squisq-capture-root') as HTMLElement;
+    if (!root) {
+      throw new Error('Capture root element not found');
+    }
+
+    // Render the DOM to a canvas via html2canvas.
+    // We're in the same document (no iframe), so COEP doesn't block cloning.
+    const canvas = await html2canvas(root, {
+      width,
+      height,
+      scale: 1,
+      useCORS: true,
+      allowTaint: true,
+      backgroundColor: '#000000',
+      logging: false,
+    });
+
+    // Convert to ImageBitmap (transferable to worker — zero-copy)
+    const bitmap = await createImageBitmap(canvas);
+    return bitmap;
+  }, []);
+
+  const destroy = useCallback(() => {
+    if (rootRef.current) {
+      rootRef.current.unmount();
+      rootRef.current = null;
+    }
+    if (containerRef.current) {
+      containerRef.current.remove();
+      containerRef.current = null;
+    }
+  }, []);
+
+  // Return a stable object to prevent useEffect cleanup loops
+  // in consumers that depend on the handle reference.
+  return useMemo(() => ({ init, captureFrame, destroy }), [init, captureFrame, destroy]);
+}

package/src/hooks/useVideoExport.ts

@@ -0,0 +1,343 @@
+/**
+ * useVideoExport — Main orchestration hook for browser video export.
+ *
+ * Coordinates frame capture (hidden iframe + html2canvas) with
+ * main-thread WebCodecs encoding via mp4-muxer. Manages the full
+ * lifecycle: prepare → capture + encode → download.
+ *
+ * Encoding runs on the main thread because frame capture via html2canvas
+ * (~100-200ms per frame) is the bottleneck, not encoding (~1ms per frame
+ * with hardware-accelerated WebCodecs). Worker offloading would add
+ * complexity with minimal benefit.
+ *
+ * Usage:
+ *   const { state, progress, phase, startExport, cancel, downloadUrl } = useVideoExport();
+ *   <button onClick={() => startExport(doc, options)}>Export</button>
+ */
+
+import { useState, useRef, useCallback, useEffect } from 'react';
+import type { Doc } from '@bendyline/squisq/schemas';
+import type { MediaProvider } from '@bendyline/squisq/schemas';
+import type { VideoQuality, VideoOrientation } from '@bendyline/squisq-video';
+import { resolveDimensions } from '@bendyline/squisq-video';
+import type { CaptionMode } from '@bendyline/squisq-react';
+import { createEncoder, supportsWebCodecs, type MainThreadEncoder } from '../mainThreadEncoder.js';
+import { useFrameCapture } from './useFrameCapture.js';
+
+// ── Types ──────────────────────────────────────────────────────────
+
+export type VideoExportState =
+  | 'idle'
+  | 'preparing'
+  | 'capturing'
+  | 'encoding'
+  | 'complete'
+  | 'error';
+
+export interface VideoExportConfig {
+  /** Encoding quality preset (default: 'normal') */
+  quality?: VideoQuality;
+  /** Frames per second (default: 30) */
+  fps?: number;
+  /** Viewport orientation (default: 'landscape') */
+  orientation?: VideoOrientation;
+  /**
+   * Map of relative image paths to binary data.
+   * Used to embed images into the render HTML.
+   */
+  images?: Map<string, ArrayBuffer>;
+  /**
+   * Map of audio segment names to binary data.
+   * Used to embed audio into the render HTML.
+   */
+  audio?: Map<string, ArrayBuffer>;
+  /** MediaProvider to resolve media URLs (alternative to passing images directly) */
+  mediaProvider?: MediaProvider;
+  /** Caption mode for the exported video (default: 'off') */
+  captionMode?: CaptionMode;
+  /** Player IIFE bundle (unused in browser export, kept for CLI/Playwright path) */
+  playerScript?: string;
+}
+
+export interface VideoExportResult {
+  /** Current export state */
+  state: VideoExportState;
+  /** 0–100 progress percentage */
+  progress: number;
+  /** Human-readable description of the current phase */
+  phase: string;
+  /** Video duration detected from the doc (seconds) */
+  duration: number;
+  /** Encoder backend ('webcodecs' when active, null when idle) */
+  backend: 'webcodecs' | null;
+  /** Blob download URL (populated when state === 'complete') */
+  downloadUrl: string | null;
+  /** File size in bytes (populated when state === 'complete') */
+  fileSize: number;
+  /** Error message (populated when state === 'error') */
+  error: string | null;
+  /** Seconds elapsed since export started */
+  elapsed: number;
+  /** Estimated seconds remaining (0 when idle or complete) */
+  estimatedRemaining: number;
+  /** Start a new export */
+  startExport: (doc: Doc, config: VideoExportConfig) => Promise<void>;
+  /** Cancel an in-progress export */
+  cancel: () => void;
+  /** Reset state back to idle (e.g., after complete or error) */
+  reset: () => void;
+}
+
+// ── Hook ───────────────────────────────────────────────────────────
+
+export function useVideoExport(): VideoExportResult {
+  const [state, setState] = useState<VideoExportState>('idle');
+  const [progress, setProgress] = useState(0);
+  const [phase, setPhase] = useState('');
+  const [duration, setDuration] = useState(0);
+  const [backend, setBackend] = useState<'webcodecs' | null>(null);
+  const [downloadUrl, setDownloadUrl] = useState<string | null>(null);
+  const [fileSize, setFileSize] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+
+  const [elapsed, setElapsed] = useState(0);
+  const [estimatedRemaining, setEstimatedRemaining] = useState(0);
+
+  const encoderRef = useRef<MainThreadEncoder | null>(null);
+  const cancelledRef = useRef(false);
+  const downloadUrlRef = useRef<string | null>(null);
+  const startTimeRef = useRef<number>(0);
+  const elapsedTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+
+  const frameCapture = useFrameCapture();
+
+  // Clean up on unmount
+  useEffect(() => {
+    return () => {
+      if (elapsedTimerRef.current) clearInterval(elapsedTimerRef.current);
+      if (downloadUrlRef.current) {
+        URL.revokeObjectURL(downloadUrlRef.current);
+      }
+      if (encoderRef.current) {
+        encoderRef.current.close();
+      }
+      frameCapture.destroy();
+    };
+  }, [frameCapture]);
+
+  const reset = useCallback(() => {
+    if (downloadUrlRef.current) {
+      URL.revokeObjectURL(downloadUrlRef.current);
+      downloadUrlRef.current = null;
+    }
+    if (encoderRef.current) {
+      encoderRef.current.close();
+      encoderRef.current = null;
+    }
+    frameCapture.destroy();
+    setState('idle');
+    setProgress(0);
+    setPhase('');
+    setDuration(0);
+    setBackend(null);
+    setDownloadUrl(null);
+    setFileSize(0);
+    setError(null);
+    setElapsed(0);
+    setEstimatedRemaining(0);
+    if (elapsedTimerRef.current) clearInterval(elapsedTimerRef.current);
+    cancelledRef.current = false;
+  }, [frameCapture]);
+
+  const cancel = useCallback(() => {
+    cancelledRef.current = true;
+    if (elapsedTimerRef.current) clearInterval(elapsedTimerRef.current);
+    if (encoderRef.current) {
+      encoderRef.current.close();
+      encoderRef.current = null;
+    }
+    frameCapture.destroy();
+    setState('idle');
+    setProgress(0);
+    setPhase('Cancelled');
+  }, [frameCapture]);
+
+  const startExport = useCallback(
+    async (doc: Doc, config: VideoExportConfig) => {
+      // Clear previous state
+      cancelledRef.current = false;
+      if (downloadUrlRef.current) {
+        URL.revokeObjectURL(downloadUrlRef.current);
+        downloadUrlRef.current = null;
+      }
+      setDownloadUrl(null);
+      setFileSize(0);
+      setError(null);
+
+      const quality = config.quality ?? 'normal';
+      const fps = config.fps ?? 30;
+      const orientation = config.orientation ?? 'landscape';
+      const { width, height } = resolveDimensions({ orientation });
+
+      try {
+        // ── Check browser support ─────────────────────────────────
+        if (!supportsWebCodecs()) {
+          throw new Error(
+            'WebCodecs is not available in this browser. ' +
+              'Video export requires Chrome 94+, Edge 94+, or another Chromium-based browser.',
+          );
+        }
+
+        // ── Step 1: Prepare ───────────────────────────────────────
+        setState('preparing');
+        setPhase('Loading document…');
+        setProgress(0);
+        setElapsed(0);
+        setEstimatedRemaining(0);
+
+        // Start elapsed timer
+        startTimeRef.current = performance.now();
+        if (elapsedTimerRef.current) clearInterval(elapsedTimerRef.current);
+        elapsedTimerRef.current = setInterval(() => {
+          setElapsed(Math.floor((performance.now() - startTimeRef.current) / 1000));
+        }, 1000);
+
+        // Collect images from MediaProvider if provided and images not passed directly
+        let images = config.images;
+        if (!images && config.mediaProvider) {
+          images = new Map<string, ArrayBuffer>();
+          const entries = await config.mediaProvider.listMedia();
+          for (const entry of entries) {
+            const url = await config.mediaProvider.resolveUrl(entry.name);
+            const res = await fetch(url);
+            if (res.ok) {
+              const data = await res.arrayBuffer();
+              images.set(entry.name, data);
+            }
+          }
+        }
+
+        const docDuration = await frameCapture.init(
+          doc,
+          { images, audio: config.audio, width, height },
+          config.captionMode,
+        );
+
+        if (cancelledRef.current) return;
+
+        setDuration(docDuration);
+        if (docDuration <= 0) {
+          throw new Error('Document has zero duration — nothing to export');
+        }
+
+        // ── Step 2: Create encoder ────────────────────────────────
+        setPhase('Starting encoder…');
+        setProgress(5);
+
+        const encoder = createEncoder({ width, height, fps, quality });
+        encoderRef.current = encoder;
+        setBackend('webcodecs');
+
+        if (cancelledRef.current) return;
+
+        // ── Step 3: Capture frames and encode ─────────────────────
+        setState('capturing');
+        const totalFrames = Math.ceil(docDuration * fps);
+
+        const captureStartTime = performance.now();
+        // Throttle UI updates to every ~10 frames to avoid excessive re-renders.
+        // Each setState between awaits triggers a separate render cycle.
+        const UI_UPDATE_INTERVAL = 10;
+
+        for (let i = 0; i < totalFrames; i++) {
+          if (cancelledRef.current) return;
+
+          const time = i / fps;
+
+          // Update UI periodically (not every frame)
+          if (i % UI_UPDATE_INTERVAL === 0 || i === totalFrames - 1) {
+            const captureProgress = Math.round((i / totalFrames) * 90);
+            setProgress(5 + captureProgress);
+            setPhase(`Capturing frame ${i + 1}/${totalFrames} (${time.toFixed(1)}s)`);
+
+            if (i > 0) {
+              const elapsedCapture = (performance.now() - captureStartTime) / 1000;
+              const avgPerFrame = elapsedCapture / i;
+              const remaining = Math.round(avgPerFrame * (totalFrames - i));
+              setEstimatedRemaining(remaining);
+            }
+          }
+
+          const bitmap = await frameCapture.captureFrame(time);
+
+          if (cancelledRef.current) {
+            bitmap.close();
+            return;
+          }
+
+          // Encode immediately — WebCodecs is fast and async internally
+          encoder.encodeFrame(bitmap, i);
+        }
+
+        if (cancelledRef.current) return;
+
+        // ── Step 4: Finalize MP4 ──────────────────────────────────
+        setState('encoding');
+        setPhase('Finalizing video…');
+        setProgress(95);
+
+        const mp4Buffer = await encoder.finalize();
+        encoderRef.current = null;
+
+        if (cancelledRef.current) return;
+
+        // ── Step 5: Create download URL ───────────────────────────
+        const blob = new Blob([mp4Buffer], { type: 'video/mp4' });
+        const url = URL.createObjectURL(blob);
+        downloadUrlRef.current = url;
+
+        setDownloadUrl(url);
+        setFileSize(mp4Buffer.byteLength);
+        setState('complete');
+        setProgress(100);
+        setPhase('Export complete');
+        setEstimatedRemaining(0);
+        if (elapsedTimerRef.current) clearInterval(elapsedTimerRef.current);
+
+        // Clean up
+        frameCapture.destroy();
+      } catch (err: unknown) {
+        if (elapsedTimerRef.current) clearInterval(elapsedTimerRef.current);
+        if (cancelledRef.current) return;
+        const message = err instanceof Error ? err.message : String(err);
+        setState('error');
+        setError(message);
+        setPhase('Export failed');
+
+        // Clean up on error
+        if (encoderRef.current) {
+          encoderRef.current.close();
+          encoderRef.current = null;
+        }
+        frameCapture.destroy();
+      }
+    },
+    [frameCapture],
+  );
+
+  return {
+    state,
+    progress,
+    phase,
+    duration,
+    backend,
+    downloadUrl,
+    fileSize,
+    error,
+    elapsed,
+    estimatedRemaining,
+    startExport,
+    cancel,
+    reset,
+  };
+}

package/src/index.ts

@@ -0,0 +1,35 @@
+/**
+ * @bendyline/squisq-video-react — Browser Video Export for Squisq Documents
+ *
+ * Provides React components and hooks for exporting Squisq documents
+ * to MP4 video directly in the browser.
+ *
+ * - VideoExportModal: Full modal UI for configure → export → download
+ * - VideoExportButton: Drop-in button that opens the modal
+ * - useVideoExport: Low-level hook for custom UIs
+ * - useFrameCapture: Frame capture via hidden iframe + html2canvas
+ *
+ * Encoding: WebCodecs (H.264 via hardware-accelerated VideoEncoder, Chrome 94+)
+ */
+
+// ── Components ─────────────────────────────────────────────────────
+export { VideoExportModal } from './VideoExportModal.js';
+export type { VideoExportModalProps } from './VideoExportModal.js';
+
+export { VideoExportButton } from './VideoExportButton.js';
+export type { VideoExportButtonProps } from './VideoExportButton.js';
+
+// ── Hooks ──────────────────────────────────────────────────────────
+export { useVideoExport } from './hooks/useVideoExport.js';
+export type {
+  VideoExportState,
+  VideoExportConfig,
+  VideoExportResult,
+} from './hooks/useVideoExport.js';
+
+export { useFrameCapture } from './hooks/useFrameCapture.js';
+export type { FrameCaptureHandle } from './hooks/useFrameCapture.js';
+
+// ── Encoder Utilities (for advanced usage) ─────────────────────────
+export { supportsWebCodecs, createEncoder } from './mainThreadEncoder.js';
+export type { MainThreadEncoder } from './mainThreadEncoder.js';