@bendyline/squisq-video-react 1.1.2 → 1.2.1

package/README.md

@@ -1,6 +1,6 @@
 # @bendyline/squisq-video-react
 
-React components and hooks for exporting Squisq documents to MP4 video directly in the browser. Uses WebCodecs for hardware-accelerated H.264 encoding with html2canvas for frame capture.
+React components and hooks for exporting Squisq documents to MP4 video directly in the browser. Uses WebCodecs for hardware-accelerated H.264 encoding (with an ffmpeg.wasm worker fallback) and html2canvas for frame capture. As of v1.5 the exported MP4 also carries an **audio** track (narration + timed media).
 
 Part of the [Squisq](https://github.com/bendyline/squisq) monorepo.
 
@@ -27,6 +27,11 @@ function App() {
 }
 ```
 
+**v1.5:** `playerScript` is now **optional** — the browser export captures frames
+from a live in-page `DocPlayer`, so the standalone bundle is only needed for
+CLI/Playwright-style pipelines. A new `defaultConfig?: Partial<VideoExportConfig>`
+prop seeds the modal's initial quality/fps/orientation/caption selections.
+
 ### Full Export Modal
 
 ```tsx
@@ -85,10 +90,13 @@ function CustomExport({ doc, images, audio }) {
   const {
     state, // 'idle' | 'preparing' | 'capturing' | 'encoding' | 'complete' | 'error'
     progress, // 0–100
+    backend, // 'webcodecs' | 'ffmpeg-wasm' | null
     elapsed,
     estimatedRemaining,
     downloadUrl,
     fileSize,
+    audioIncluded, // whether an audio track was muxed in
+    audioSkippedReason, // null when the doc had no audio; a string explains a shortfall
     error,
     startExport,
     cancel,
@@ -97,7 +105,7 @@ function CustomExport({ doc, images, audio }) {
 
   return (
     <div>
-      <button onClick={() => startExport({ doc, images, audio, quality: 'normal', fps: 30 })}>
+      <button onClick={() => startExport(doc, { images, audio, quality: 'normal', fps: 30 })}>
         Export
       </button>
       {state === 'capturing' && <p>Progress: {progress}%</p>}
@@ -113,16 +121,34 @@ function CustomExport({ doc, images, audio }) {
 
 ## Browser Requirements
 
-WebCodecs H.264 encoding requires Chrome 94+ or Edge 94+. Use `supportsWebCodecs()` to check at runtime:
+WebCodecs H.264 encoding requires Chrome 94+ or Edge 94+. When WebCodecs H.264
+is unavailable, the export automatically falls back to an ffmpeg.wasm worker —
+which requires `SharedArrayBuffer` (i.e. Cross-Origin-Isolation headers on the
+host page). Use `supportsWebCodecs()` to probe at runtime:
 
 ```ts
-import { supportsWebCodecs } from '@bendyline/squisq-video-react';
+import {
+  supportsWebCodecs,
+  supportsWebCodecsH264,
+  supportsWebCodecsAac,
+} from '@bendyline/squisq-video-react';
 
 if (!supportsWebCodecs()) {
-  // Fall back or show unsupported message
+  // ffmpeg.wasm fallback will be used (needs Cross-Origin-Isolation)
 }
 ```
 
+**Audio tiers.** The audio track is muxed via WebCodecs AAC when available
+(`supportsWebCodecsAac()`); otherwise it is skipped and the export reports
+`audioIncluded: false` with an `audioSkippedReason`. Audio problems never fail
+the export — the video always completes. `supportsWebCodecsH264(config)` probes a
+specific encoder configuration; `EncoderConfig` is also exported.
+
+## Full API Reference
+
+See [docs/API.md](https://github.com/bendyline/squisq/blob/main/docs/API.md)
+for complete prop tables, `VideoExportConfig`, and the encoder utilities.
+
 ## Related Packages
 
 | Package                                                                          | Description                                     |

package/dist/chunk-6DKLHXX3.js

@@ -0,0 +1,47 @@
+// src/mp4Mux.ts
+import { Muxer, ArrayBufferTarget } from "mp4-muxer";
+function createMp4Muxer(options) {
+  const target = new ArrayBufferTarget();
+  const muxer = new Muxer({
+    target,
+    video: {
+      codec: "avc",
+      width: options.width,
+      height: options.height
+    },
+    // Only declare the audio track when requested so the video-only
+    // configuration stays byte-identical to the historical output.
+    ...options.audio ? {
+      audio: {
+        codec: "aac",
+        numberOfChannels: options.audio.numberOfChannels,
+        sampleRate: options.audio.sampleRate
+      }
+    } : {},
+    fastStart: "in-memory"
+  });
+  return {
+    hasAudioTrack: options.audio !== void 0,
+    addVideoChunk(chunk, meta) {
+      muxer.addVideoChunk(chunk, meta);
+    },
+    addVideoChunkRaw(data, type, timestampMicros, durationMicros, meta) {
+      muxer.addVideoChunkRaw(data, type, timestampMicros, durationMicros, meta);
+    },
+    addAudioChunk(chunk, meta) {
+      muxer.addAudioChunk(chunk, meta);
+    },
+    addAudioChunkRaw(data, type, timestampMicros, durationMicros, meta) {
+      muxer.addAudioChunkRaw(data, type, timestampMicros, durationMicros, meta);
+    },
+    finalize() {
+      muxer.finalize();
+      return target.buffer;
+    }
+  };
+}
+
+export {
+  createMp4Muxer
+};
+//# sourceMappingURL=chunk-6DKLHXX3.js.map

package/dist/chunk-6DKLHXX3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/mp4Mux.ts"],"sourcesContent":["/**\n * mp4Mux — Thin wrapper around mp4-muxer for WebCodecs encoding.\n *\n * Creates a Muxer instance configured for H.264 video (and, optionally, an\n * AAC audio track), accumulates encoded chunks, and produces a final MP4\n * ArrayBuffer. When no `audio` option is supplied the muxer configuration is\n * byte-identical to the video-only path.\n */\n\nimport { Muxer, ArrayBufferTarget } from 'mp4-muxer';\n\nexport interface Mp4MuxerOptions {\n  width: number;\n  height: number;\n  fps: number;\n  /**\n   * When present, declares an AAC audio track alongside the video track.\n   * Feed encoded audio via {@link Mp4MuxerHandle.addAudioChunk} (from a\n   * WebCodecs `AudioEncoder`) or {@link Mp4MuxerHandle.addAudioChunkRaw}.\n   */\n  audio?: {\n    numberOfChannels: number;\n    sampleRate: number;\n  };\n}\n\nexport interface Mp4MuxerHandle {\n  /** Add an encoded video chunk to the muxer. */\n  addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): void;\n  /**\n   * Add a raw video sample (bytes + timing) without an `EncodedVideoChunk`.\n   * Useful where no `VideoEncoder` instance is available (e.g. Node tests).\n   */\n  addVideoChunkRaw(\n    data: Uint8Array,\n    type: 'key' | 'delta',\n    timestampMicros: number,\n    durationMicros: number,\n    meta?: EncodedVideoChunkMetadata,\n  ): void;\n  /** Add an encoded audio chunk (from a WebCodecs `AudioEncoder`). */\n  addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void;\n  /**\n   * Add a raw audio sample (bytes + timing) without an `EncodedAudioChunk`.\n   * Useful where no `AudioEncoder` instance is available (e.g. Node tests).\n   */\n  addAudioChunkRaw(\n    data: Uint8Array,\n    type: 'key' | 'delta',\n    timestampMicros: number,\n    durationMicros: number,\n    meta?: EncodedAudioChunkMetadata,\n  ): void;\n  /** Whether this muxer was created with an audio track. */\n  readonly hasAudioTrack: boolean;\n  /** Finalize and return the MP4 as an ArrayBuffer. */\n  finalize(): ArrayBuffer;\n}\n\n/**\n * Create an MP4 muxer configured for H.264 video, plus an optional AAC track.\n */\nexport function createMp4Muxer(options: Mp4MuxerOptions): Mp4MuxerHandle {\n  const target = new ArrayBufferTarget();\n\n  const muxer = new Muxer({\n    target,\n    video: {\n      codec: 'avc',\n      width: options.width,\n      height: options.height,\n    },\n    // Only declare the audio track when requested so the video-only\n    // configuration stays byte-identical to the historical output.\n    ...(options.audio\n      ? {\n          audio: {\n            codec: 'aac' as const,\n            numberOfChannels: options.audio.numberOfChannels,\n            sampleRate: options.audio.sampleRate,\n          },\n        }\n      : {}),\n    fastStart: 'in-memory',\n  });\n\n  return {\n    hasAudioTrack: options.audio !== undefined,\n\n    addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) {\n      muxer.addVideoChunk(chunk, meta);\n    },\n\n    addVideoChunkRaw(\n      data: Uint8Array,\n      type: 'key' | 'delta',\n      timestampMicros: number,\n      durationMicros: number,\n      meta?: EncodedVideoChunkMetadata,\n    ) {\n      muxer.addVideoChunkRaw(data, type, timestampMicros, durationMicros, meta);\n    },\n\n    addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) {\n      muxer.addAudioChunk(chunk, meta);\n    },\n\n    addAudioChunkRaw(\n      data: Uint8Array,\n      type: 'key' | 'delta',\n      timestampMicros: number,\n      durationMicros: number,\n      meta?: EncodedAudioChunkMetadata,\n    ) {\n      muxer.addAudioChunkRaw(data, type, timestampMicros, durationMicros, meta);\n    },\n\n    finalize(): ArrayBuffer {\n      muxer.finalize();\n      return target.buffer;\n    },\n  };\n}\n"],"mappings":";AASA,SAAS,OAAO,yBAAyB;AAqDlC,SAAS,eAAe,SAA0C;AACvE,QAAM,SAAS,IAAI,kBAAkB;AAErC,QAAM,QAAQ,IAAI,MAAM;AAAA,IACtB;AAAA,IACA,OAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO,QAAQ;AAAA,MACf,QAAQ,QAAQ;AAAA,IAClB;AAAA;AAAA;AAAA,IAGA,GAAI,QAAQ,QACR;AAAA,MACE,OAAO;AAAA,QACL,OAAO;AAAA,QACP,kBAAkB,QAAQ,MAAM;AAAA,QAChC,YAAY,QAAQ,MAAM;AAAA,MAC5B;AAAA,IACF,IACA,CAAC;AAAA,IACL,WAAW;AAAA,EACb,CAAC;AAED,SAAO;AAAA,IACL,eAAe,QAAQ,UAAU;AAAA,IAEjC,cAAc,OAA0B,MAAkC;AACxE,YAAM,cAAc,OAAO,IAAI;AAAA,IACjC;AAAA,IAEA,iBACE,MACA,MACA,iBACA,gBACA,MACA;AACA,YAAM,iBAAiB,MAAM,MAAM,iBAAiB,gBAAgB,IAAI;AAAA,IAC1E;AAAA,IAEA,cAAc,OAA0B,MAAkC;AACxE,YAAM,cAAc,OAAO,IAAI;AAAA,IACjC;AAAA,IAEA,iBACE,MACA,MACA,iBACA,gBACA,MACA;AACA,YAAM,iBAAiB,MAAM,MAAM,iBAAiB,gBAAgB,IAAI;AAAA,IAC1E;AAAA,IAEA,WAAwB;AACtB,YAAM,SAAS;AACf,aAAO,OAAO;AAAA,IAChB;AAAA,EACF;AACF;","names":[]}

package/dist/index.d.ts

@@ -1,44 +1,8 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
-import { Doc, MediaProvider } from '@bendyline/squisq/schemas';
+import { MediaProvider, Doc } from '@bendyline/squisq/schemas';
 import { VideoQuality, VideoOrientation, RenderHtmlOptions } from '@bendyline/squisq-video';
 import { CaptionMode } from '@bendyline/squisq-react';
 
-interface VideoExportModalProps {
-    /** The document to export */
-    doc: Doc;
-    /** Player IIFE bundle source */
-    playerScript: string;
-    /** Optional media provider for resolving images/audio */
-    mediaProvider?: MediaProvider;
-    /** Pre-collected images map (alternative to mediaProvider) */
-    images?: Map<string, ArrayBuffer>;
-    /** Pre-collected audio map */
-    audio?: Map<string, ArrayBuffer>;
-    /** Called when the modal should close */
-    onClose: () => void;
-}
-declare function VideoExportModal({ doc, playerScript, mediaProvider, images, audio, onClose, }: VideoExportModalProps): react_jsx_runtime.JSX.Element;
-
-interface VideoExportButtonProps {
-    /** The document to export */
-    doc: Doc;
-    /** Player IIFE bundle source */
-    playerScript: string;
-    /** Optional media provider for resolving images/audio */
-    mediaProvider?: MediaProvider;
-    /** Pre-collected images map */
-    images?: Map<string, ArrayBuffer>;
-    /** Pre-collected audio map */
-    audio?: Map<string, ArrayBuffer>;
-    /** Button label (default: "Export Video") */
-    label?: string;
-    /** Additional inline styles for the button */
-    style?: React.CSSProperties;
-    /** Whether the button is disabled */
-    disabled?: boolean;
-}
-declare function VideoExportButton({ doc, playerScript, mediaProvider, images, audio, label, style, disabled, }: VideoExportButtonProps): react_jsx_runtime.JSX.Element;
-
 /**
  * useVideoExport — Main orchestration hook for browser video export.
  *
@@ -96,6 +60,19 @@ interface VideoExportResult {
     downloadUrl: string | null;
     /** File size in bytes (populated when state === 'complete') */
     fileSize: number;
+    /**
+     * Whether an audio track was muxed into the exported MP4 (populated when
+     * state === 'complete'). False when the doc had no audio or when audio was
+     * skipped/degraded — see {@link audioSkippedReason}.
+     */
+    audioIncluded: boolean;
+    /**
+     * Why audio is absent, when `audioIncluded` is false. `null` means the doc
+     * simply had no audio (not a failure); a non-null string explains a genuine
+     * capability shortfall or runtime failure. Audio problems never fail the
+     * export — the video always completes.
+     */
+    audioSkippedReason: string | null;
     /** Error message (populated when state === 'error') */
     error: string | null;
     /** Seconds elapsed since export started */
@@ -111,6 +88,63 @@ interface VideoExportResult {
 }
 declare function useVideoExport(): VideoExportResult;
 
+interface VideoExportModalProps {
+    /** The document to export */
+    doc: Doc;
+    /**
+     * Player IIFE bundle source. Unused by the browser export path (frames
+     * are captured from a live in-page DocPlayer); only forwarded for
+     * CLI/Playwright-style pipelines that render standalone HTML.
+     */
+    playerScript?: string;
+    /** Optional media provider for resolving images/audio */
+    mediaProvider?: MediaProvider;
+    /** Pre-collected images map (alternative to mediaProvider) */
+    images?: Map<string, ArrayBuffer>;
+    /** Pre-collected audio map */
+    audio?: Map<string, ArrayBuffer>;
+    /**
+     * Seeds the modal's initial quality/fps/orientation/captionMode selections
+     * and is merged (as a base) into the config passed to the export hook, so a
+     * host can share one config shape with `useVideoExport`. The individual
+     * `images`/`audio`/`mediaProvider`/`playerScript` props still take
+     * precedence over any matching key here.
+     */
+    defaultConfig?: Partial<VideoExportConfig>;
+    /** Called when the modal should close */
+    onClose: () => void;
+}
+declare function VideoExportModal({ doc, playerScript, mediaProvider, images, audio, defaultConfig, onClose, }: VideoExportModalProps): react_jsx_runtime.JSX.Element;
+
+interface VideoExportButtonProps {
+    /** The document to export */
+    doc: Doc;
+    /**
+     * Player IIFE bundle source. Unused by the browser export path (frames
+     * are captured from a live in-page DocPlayer); only forwarded for
+     * CLI/Playwright-style pipelines that render standalone HTML.
+     */
+    playerScript?: string;
+    /** Optional media provider for resolving images/audio */
+    mediaProvider?: MediaProvider;
+    /** Pre-collected images map */
+    images?: Map<string, ArrayBuffer>;
+    /** Pre-collected audio map */
+    audio?: Map<string, ArrayBuffer>;
+    /**
+     * Seeds the modal's initial export settings and is merged as a base into the
+     * config passed to the export hook. Forwarded to {@link VideoExportModal}.
+     */
+    defaultConfig?: Partial<VideoExportConfig>;
+    /** Button label (default: "Export Video") */
+    label?: string;
+    /** Additional inline styles for the button */
+    style?: React.CSSProperties;
+    /** Whether the button is disabled */
+    disabled?: boolean;
+}
+declare function VideoExportButton({ doc, playerScript, mediaProvider, images, audio, defaultConfig, label, style, disabled, }: VideoExportButtonProps): react_jsx_runtime.JSX.Element;
+
 /**
  * useFrameCapture — Hidden div + html2canvas frame capture.
  *
@@ -153,10 +187,25 @@ interface EncoderConfig {
     height: number;
     fps: number;
     quality: 'draft' | 'normal' | 'high';
+    /**
+     * When present, the underlying muxer is configured with an AAC audio track
+     * and {@link MainThreadEncoder.addAudioChunk} becomes usable. Absent → the
+     * encoder produces a video-only MP4 exactly as before.
+     */
+    audio?: {
+        numberOfChannels: number;
+        sampleRate: number;
+    };
 }
 interface MainThreadEncoder {
     /** Encode a single frame. The bitmap is closed after encoding. */
     encodeFrame(bitmap: ImageBitmap, frameIndex: number): void;
+    /**
+     * Hand an encoded audio chunk (from a WebCodecs `AudioEncoder`) to the muxer.
+     * Only valid when the encoder was created with an `audio` config; otherwise a
+     * no-op. Must be called before {@link finalize}.
+     */
+    addAudioChunk?(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void;
     /** Flush pending frames and finalize the MP4. Returns the MP4 ArrayBuffer. */
     finalize(): Promise<ArrayBuffer>;
     /** Close the encoder without producing output (e.g., on cancel). */
@@ -166,6 +215,13 @@ interface MainThreadEncoder {
  * Check whether the browser supports WebCodecs video encoding.
  */
 declare function supportsWebCodecs(): boolean;
+/**
+ * Probe whether the WebCodecs encoder actually supports the H.264 profile
+ * we use. The `VideoEncoder` global can exist while the specific codec is
+ * unavailable — this is the case on Linux Chromium, which ships without
+ * the proprietary H.264 encoder.
+ */
+declare function supportsWebCodecsH264(config: EncoderConfig): Promise<boolean>;
 /**
  * Create a main-thread WebCodecs encoder.
  *
@@ -173,4 +229,30 @@ declare function supportsWebCodecs(): boolean;
  */
 declare function createEncoder(config: EncoderConfig): MainThreadEncoder;
 
-export { type FrameCaptureHandle, type MainThreadEncoder, VideoExportButton, type VideoExportButtonProps, type VideoExportConfig, VideoExportModal, type VideoExportModalProps, type VideoExportResult, type VideoExportState, createEncoder, supportsWebCodecs, useFrameCapture, useVideoExport };
+/**
+ * audioTrack — In-browser audio rendering + AAC encoding for MP4 export.
+ *
+ * Replaces the CLI's ffmpeg `adelay`/`atrim`/`amix` graph with browser-native
+ * primitives:
+ *
+ *  - {@link supportsWebCodecsAac} probes whether the browser can encode AAC
+ *    (WebCodecs `AudioEncoder`).
+ *  - {@link renderAudioTimeline} mixes an {@link AudioTimelineClip} list into a
+ *    single `AudioBuffer` using an `OfflineAudioContext` (decode → schedule →
+ *    render).
+ *  - {@link encodeAacTrack} feeds that buffer through a WebCodecs
+ *    `AudioEncoder` and hands the chunks to an mp4-muxer audio track (tier 1).
+ *  - {@link audioBufferToWav} + {@link muxAudioWithFfmpegWasm} cover the
+ *    ffmpeg.wasm fallback (tier 2).
+ *  - {@link selectAudioTier} is the pure capability→tier decision shared with
+ *    `useVideoExport` (and unit-tested at the module boundary).
+ */
+
+/**
+ * Whether the browser can encode AAC (`mp4a.40.2`) via WebCodecs.
+ * Returns false when `AudioEncoder` is undefined (e.g. Node/jsdom, Safari,
+ * older Chromium) or the config is unsupported.
+ */
+declare function supportsWebCodecsAac(sampleRate?: number, channels?: number): Promise<boolean>;
+
+export { type EncoderConfig, type FrameCaptureHandle, type MainThreadEncoder, VideoExportButton, type VideoExportButtonProps, type VideoExportConfig, VideoExportModal, type VideoExportModalProps, type VideoExportResult, type VideoExportState, createEncoder, supportsWebCodecs, supportsWebCodecsAac, supportsWebCodecsH264, useFrameCapture, useVideoExport };