@twick/browser-render 0.15.7 → 0.15.8

package/README.md

@@ -46,19 +46,21 @@ a.click();
 import { useBrowserRenderer } from '@twick/browser-render';
 
 function VideoRenderer() {
-  const { render, progress, isRendering, videoBlob, download } = useBrowserRenderer({
-    width: 1920,
-    height: 1080,
-    fps: 30,
-    autoDownload: false
-  });
+  const { render, progress, isRendering, videoBlob, download, error, reset } =
+    useBrowserRenderer({
+      width: 720,
+      height: 1280,
+      fps: 30,
+      includeAudio: true,   // enable audio rendering + FFmpeg mux
+      autoDownload: true,   // auto-download final MP4
+    });
 
   const handleRender = async () => {
     await render({
       input: {
-        properties: { width: 1920, height: 1080, fps: 30 },
-        tracks: [/* ... */]
-      }
+        properties: { width: 720, height: 1280, fps: 30 },
+        tracks: [/* ... */],
+      },
     });
   };
 
@@ -69,6 +71,12 @@ function VideoRenderer() {
       </button>
       {isRendering && <progress value={progress} max={1} />}
       {videoBlob && <button onClick={download}>Download</button>}
+      {error && (
+        <div>
+          <p>{error.message}</p>
+          <button onClick={reset}>Clear error</button>
+        </div>
+      )}
     </div>
   );
 }
@@ -92,6 +100,9 @@ function VideoRenderer() {
     fps?: number;            // Frames per second (default: 30)
     quality?: 'low' | 'medium' | 'high';
     range?: [number, number]; // [start, end] in seconds
+    includeAudio?: boolean;  // Enable audio rendering and muxing (default: false)
+    downloadAudioSeparately?: boolean; // Also download audio.wav when muxing fails
+    onAudioReady?: (audioBlob: Blob) => void; // Callback when raw audio is ready
     onProgress?: (progress: number) => void;
     onComplete?: (videoBlob: Blob) => void;
     onError?: (error: Error) => void;
@@ -130,9 +141,38 @@ export default defineConfig({
 });
 ```
 
+### FFmpeg audio/video muxing (optional)
+
+When `settings.includeAudio` is enabled, `@twick/browser-render` will render audio in the browser and (by default) try to mux it into the final MP4 using `@ffmpeg/ffmpeg`.  
+To match the setup used in `@twick/examples`, you should:
+
+1. Install the FFmpeg packages:
+
+```bash
+npm install @ffmpeg/ffmpeg @ffmpeg/util @ffmpeg/core
+```
+
+2. Expose the FFmpeg core files from your app’s `public` folder so they are available at:
+
+```text
+/public/ffmpeg/ffmpeg-core.js
+/public/ffmpeg/ffmpeg-core.wasm
+```
+
+For example, you can copy them from `node_modules/@ffmpeg/core/dist`:
+
+```bash
+mkdir -p public/ffmpeg
+cp node_modules/@ffmpeg/core/dist/ffmpeg-core.js public/ffmpeg/
+cp node_modules/@ffmpeg/core/dist/ffmpeg-core.wasm public/ffmpeg/
+```
+
+In the browser, the muxer loads these files from `${window.location.origin}/ffmpeg`, so the URLs must match that structure.  
+If FFmpeg cannot be loaded, the renderer will fall back to returning a video-only file and (optionally) downloading `audio.wav` separately when `downloadAudioSeparately` is true.
+
 ## Limitations
 
-- **Audio**: Audio processing is not yet implemented. Only video encoding is supported.
+- **Audio**: Audio rendering and FFmpeg-based muxing run entirely in the browser and are still considered experimental. If FFmpeg assets are not available, only video will be muxed and audio may be downloaded as a separate file.
 - **Browser Support**: Requires WebCodecs API (Chrome 94+, Edge 94+)
 
 ## License

package/dist/index.js

@@ -41,6 +41,95 @@ module.exports = __toCommonJS(index_exports);
 var import_core = require("@twick/core");
 var import_project = __toESM(require("@twick/visualizer/dist/project.js"));
 
+// src/audio/video-audio-extractor.ts
+var VideoElementAudioExtractor = class {
+  audioContext;
+  video;
+  destination = null;
+  mediaRecorder = null;
+  audioChunks = [];
+  constructor(videoSrc, sampleRate = 48e3) {
+    this.audioContext = new AudioContext({ sampleRate });
+    this.video = document.createElement("video");
+    this.video.crossOrigin = "anonymous";
+    this.video.src = videoSrc;
+    this.video.muted = true;
+  }
+  async initialize() {
+    return new Promise((resolve, reject) => {
+      this.video.addEventListener("loadedmetadata", () => resolve(), { once: true });
+      this.video.addEventListener("error", (e) => {
+        reject(new Error(`Failed to load video for audio extraction: ${e}`));
+      }, { once: true });
+    });
+  }
+  /**
+   * Extract audio by playing the video and capturing audio output
+   */
+  async extractAudio(startTime, duration, playbackRate = 1) {
+    const source = this.audioContext.createMediaElementSource(this.video);
+    this.destination = this.audioContext.createMediaStreamDestination();
+    source.connect(this.destination);
+    this.audioChunks = [];
+    this.mediaRecorder = new MediaRecorder(this.destination.stream, {
+      mimeType: "audio/webm"
+    });
+    this.mediaRecorder.ondataavailable = (event) => {
+      if (event.data.size > 0) {
+        this.audioChunks.push(event.data);
+      }
+    };
+    this.video.currentTime = startTime;
+    this.video.playbackRate = playbackRate;
+    await new Promise((resolve) => {
+      this.video.addEventListener("seeked", () => resolve(), { once: true });
+    });
+    return new Promise((resolve, reject) => {
+      const recordingTimeout = setTimeout(() => {
+        reject(new Error("Audio extraction timeout"));
+      }, (duration / playbackRate + 5) * 1e3);
+      this.mediaRecorder.start();
+      this.video.play();
+      setTimeout(async () => {
+        clearTimeout(recordingTimeout);
+        this.video.pause();
+        this.mediaRecorder.stop();
+        await new Promise((res) => {
+          this.mediaRecorder.addEventListener("stop", () => res(), { once: true });
+        });
+        try {
+          const audioBlob = new Blob(this.audioChunks, { type: "audio/webm" });
+          const arrayBuffer = await audioBlob.arrayBuffer();
+          const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
+          resolve(audioBuffer);
+        } catch (err) {
+          reject(new Error(`Failed to decode recorded audio: ${err}`));
+        }
+      }, duration / playbackRate * 1e3);
+    });
+  }
+  async close() {
+    if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
+      this.mediaRecorder.stop();
+    }
+    this.video.pause();
+    this.video.src = "";
+    if (this.audioContext.state !== "closed") {
+      await this.audioContext.close();
+    }
+  }
+};
+async function extractAudioFromVideo(videoSrc, startTime, duration, playbackRate = 1, sampleRate = 48e3) {
+  const extractor = new VideoElementAudioExtractor(videoSrc, sampleRate);
+  try {
+    await extractor.initialize();
+    const audioBuffer = await extractor.extractAudio(startTime, duration, playbackRate);
+    return audioBuffer;
+  } finally {
+    await extractor.close();
+  }
+}
+
 // src/audio/audio-processor.ts
 function getAssetPlacement(frames) {
   const assets = [];
@@ -93,11 +182,26 @@ var BrowserAudioProcessor = class {
   audioContext;
   /**
    * Fetch and decode audio from a media source
+   * Falls back to video element extraction if decodeAudioData fails
    */
   async fetchAndDecodeAudio(src) {
-    const response = await fetch(src);
-    const arrayBuffer = await response.arrayBuffer();
-    return await this.audioContext.decodeAudioData(arrayBuffer);
+    try {
+      const response = await fetch(src);
+      const arrayBuffer = await response.arrayBuffer();
+      return await this.audioContext.decodeAudioData(arrayBuffer);
+    } catch (err) {
+      try {
+        return await extractAudioFromVideo(
+          src,
+          0,
+          999999,
+          1,
+          this.sampleRate
+        );
+      } catch (fallbackErr) {
+        throw new Error(`Failed to extract audio: ${err}. Fallback also failed: ${fallbackErr}`);
+      }
+    }
   }
   /**
    * Process audio asset with playback rate, volume, and timing
@@ -211,6 +315,55 @@ var BrowserAudioProcessor = class {
   }
 };
 
+// src/audio/audio-video-muxer.ts
+function getFFmpegBaseURL() {
+  if (typeof window !== "undefined") {
+    return `${window.location.origin}/ffmpeg`;
+  }
+  return "/ffmpeg";
+}
+async function muxAudioVideo(options) {
+  try {
+    const { FFmpeg } = await import("@ffmpeg/ffmpeg");
+    const { fetchFile } = await import("@ffmpeg/util");
+    const ffmpeg = new FFmpeg();
+    const base = getFFmpegBaseURL();
+    const coreURL = `${base}/ffmpeg-core.js`;
+    const wasmURL = `${base}/ffmpeg-core.wasm`;
+    await ffmpeg.load({
+      coreURL,
+      wasmURL
+    });
+    await ffmpeg.writeFile(
+      "video.mp4",
+      await fetchFile(options.videoBlob)
+    );
+    await ffmpeg.writeFile(
+      "audio.wav",
+      new Uint8Array(options.audioBuffer)
+    );
+    await ffmpeg.exec([
+      "-i",
+      "video.mp4",
+      "-i",
+      "audio.wav",
+      "-c:v",
+      "copy",
+      "-c:a",
+      "aac",
+      "-b:a",
+      "192k",
+      "-shortest",
+      "output.mp4"
+    ]);
+    const data = await ffmpeg.readFile("output.mp4");
+    const uint8 = typeof data === "string" ? new TextEncoder().encode(data) : new Uint8Array(data);
+    return new Blob([uint8], { type: "video/mp4" });
+  } catch {
+    return options.videoBlob;
+  }
+}
+
 // src/browser-renderer.ts
 var BrowserWasmExporter = class _BrowserWasmExporter {
   constructor(settings) {
@@ -271,7 +424,6 @@ var BrowserWasmExporter = class _BrowserWasmExporter {
         fps: this.fps
       });
     } catch (error) {
-      console.error("WASM loading error:", error);
       throw error;
     }
   }
@@ -294,42 +446,33 @@ var BrowserWasmExporter = class _BrowserWasmExporter {
   }
   async generateAudio(assets, startFrame, endFrame) {
     try {
-      console.log("\u{1F50A} Starting audio processing...", {
-        frames: assets.length,
-        startFrame,
-        endFrame
-      });
       const processor = new BrowserAudioProcessor();
       const assetPlacements = getAssetPlacement(assets);
-      console.log(`\u{1F4CA} Found ${assetPlacements.length} audio assets to process`);
       if (assetPlacements.length === 0) {
-        console.log("\u26A0\uFE0F No audio assets found");
         return null;
       }
       const processedBuffers = [];
       for (const asset of assetPlacements) {
         if (asset.volume > 0 && asset.playbackRate > 0) {
-          console.log(`\u{1F3B5} Processing audio: ${asset.key}`);
-          const buffer = await processor.processAudioAsset(
-            asset,
-            this.settings.fps || 30,
-            endFrame - startFrame
-          );
-          processedBuffers.push(buffer);
+          try {
+            const buffer = await processor.processAudioAsset(
+              asset,
+              this.settings.fps || 30,
+              endFrame - startFrame
+            );
+            processedBuffers.push(buffer);
+          } catch {
+          }
         }
       }
       if (processedBuffers.length === 0) {
-        console.log("\u26A0\uFE0F No audio buffers to mix");
         return null;
       }
-      console.log(`\u{1F39B}\uFE0F Mixing ${processedBuffers.length} audio track(s)...`);
       const mixedBuffer = processor.mixAudioBuffers(processedBuffers);
       const wavData = processor.audioBufferToWav(mixedBuffer);
       await processor.close();
-      console.log(`\u2705 Audio processed: ${(wavData.byteLength / 1024 / 1024).toFixed(2)} MB`);
       return wavData;
-    } catch (error) {
-      console.error("\u274C Audio processing failed:", error);
+    } catch {
       return null;
     }
   }
@@ -374,12 +517,7 @@ var renderTwickVideoInBrowser = async (config) => {
     const width = settings.width || variables.input.properties?.width || 1920;
     const height = settings.height || variables.input.properties?.height || 1080;
     const fps = settings.fps || variables.input.properties?.fps || 30;
-    let project;
-    if (!projectFile) {
-      project = import_project.default;
-    } else {
-      project = projectFile;
-    }
+    const project = !projectFile ? import_project.default : projectFile;
     project.variables = variables;
     const renderSettings = {
       name: "browser-render",
@@ -407,6 +545,46 @@ var renderTwickVideoInBrowser = async (config) => {
     renderer.playback.fps = renderSettings.fps;
     renderer.playback.state = 1;
     const totalFrames = await renderer.getNumberOfFrames(renderSettings);
+    if (totalFrames === 0 || !isFinite(totalFrames)) {
+      throw new Error(
+        "Cannot render: Video has zero duration. Please ensure your project has valid content with non-zero duration. Check that all video elements have valid sources and are properly loaded."
+      );
+    }
+    const videoElements = [];
+    if (variables.input.tracks) {
+      variables.input.tracks.forEach((track) => {
+        if (track.elements) {
+          track.elements.forEach((el) => {
+            if (el.type === "video") videoElements.push(el);
+          });
+        }
+      });
+    }
+    if (videoElements.length > 0) {
+      for (const videoEl of videoElements) {
+        const src = videoEl.props?.src;
+        if (!src || src === "undefined") continue;
+        const preloadVideo = document.createElement("video");
+        preloadVideo.crossOrigin = "anonymous";
+        preloadVideo.preload = "metadata";
+        preloadVideo.src = src;
+        await new Promise((resolve, reject) => {
+          const timeout = setTimeout(
+            () => reject(new Error(`Timeout loading video metadata: ${src.substring(0, 80)}`)),
+            3e4
+          );
+          preloadVideo.addEventListener("loadedmetadata", () => {
+            clearTimeout(timeout);
+            resolve();
+          }, { once: true });
+          preloadVideo.addEventListener("error", () => {
+            clearTimeout(timeout);
+            const err = preloadVideo.error;
+            reject(new Error(`Failed to load video: ${err?.message || "Unknown error"}`));
+          }, { once: true });
+        });
+      }
+    }
     await renderer.playback.recalculate();
     await renderer.playback.reset();
     await renderer.playback.seek(0);
@@ -423,14 +601,11 @@ var renderTwickVideoInBrowser = async (config) => {
       mediaAssets.push(currentAssets);
       const canvas = renderer.stage.finalBuffer;
       await exporter.handleFrame(canvas, frame);
-      if (settings.onProgress) {
-        settings.onProgress(frame / totalFrames);
-      }
+      if (settings.onProgress) settings.onProgress(frame / totalFrames);
     }
     await exporter.stop();
     let audioData = null;
     if (settings.includeAudio && mediaAssets.length > 0) {
-      console.log("\u{1F3B5} Generating audio track...");
       audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
     }
     let finalBlob = exporter.getVideoBlob();
@@ -438,9 +613,12 @@ var renderTwickVideoInBrowser = async (config) => {
       throw new Error("Failed to create video blob");
     }
     if (audioData && settings.includeAudio) {
-      console.log("\u2705 Audio extracted and processed successfully");
-      console.log("\u{1F4CA} Audio data size:", (audioData.byteLength / 1024 / 1024).toFixed(2), "MB");
-      if (settings.downloadAudioSeparately) {
+      try {
+        finalBlob = await muxAudioVideo({
+          videoBlob: finalBlob,
+          audioBuffer: audioData
+        });
+      } catch {
         const audioBlob = new Blob([audioData], { type: "audio/wav" });
         const audioUrl = URL.createObjectURL(audioBlob);
         const a = document.createElement("a");
@@ -448,15 +626,7 @@ var renderTwickVideoInBrowser = async (config) => {
         a.download = "audio.wav";
         a.click();
         URL.revokeObjectURL(audioUrl);
-        console.log("\u2705 Audio downloaded separately as audio.wav");
-      }
-      if (settings.onAudioReady) {
-        const audioBlob = new Blob([audioData], { type: "audio/wav" });
-        settings.onAudioReady(audioBlob);
       }
-      console.log("\u{1F4A1} Note: Client-side audio muxing is complex.");
-      console.log("\u{1F4A1} For full audio support, use server-side rendering: @twick/render-server");
-      console.log("\u{1F4A1} Or mux manually with: ffmpeg -i video.mp4 -i audio.wav -c:v copy -c:a aac output.mp4");
     }
     if (settings.onComplete) {
       settings.onComplete(finalBlob);
@@ -500,17 +670,13 @@ var useBrowserRenderer = (options = {}) => {
   }, []);
   const download = (0, import_react.useCallback)((filename) => {
     if (!videoBlob) {
-      const downloadError = new Error("No video available to download. Please render the video first.");
-      setError(downloadError);
-      console.error(downloadError.message);
+      setError(new Error("No video available to download. Please render the video first."));
       return;
     }
     try {
       downloadVideoBlob(videoBlob, filename || options.downloadFilename || "video.mp4");
     } catch (err) {
-      const downloadError = err instanceof Error ? err : new Error("Failed to download video");
-      setError(downloadError);
-      console.error("Download error:", downloadError);
+      setError(err instanceof Error ? err : new Error("Failed to download video"));
     }
   }, [videoBlob, options.downloadFilename]);
   const render = (0, import_react.useCallback)(async (variables) => {
@@ -540,9 +706,7 @@ var useBrowserRenderer = (options = {}) => {
               try {
                 downloadVideoBlob(blob2, downloadFilename || "video.mp4");
               } catch (downloadErr) {
-                const error2 = downloadErr instanceof Error ? downloadErr : new Error("Failed to auto-download video");
-                setError(error2);
-                console.error("Auto-download error:", error2);
+                setError(downloadErr instanceof Error ? downloadErr : new Error("Failed to auto-download video"));
               }
             }
           },
@@ -558,9 +722,7 @@ var useBrowserRenderer = (options = {}) => {
       setProgress(1);
       return blob;
     } catch (err) {
-      const error2 = err instanceof Error ? err : new Error(String(err));
-      setError(error2);
-      console.error("Render error:", error2);
+      setError(err instanceof Error ? err : new Error(String(err)));
       return null;
     } finally {
       setIsRendering(false);