@twick/browser-render 0.15.8 → 0.15.9

package/dist/index.mjs

@@ -1,6 +1,7 @@
 // src/browser-renderer.ts
 import { Renderer, Vector2 } from "@twick/core";
 import defaultProject from "@twick/visualizer/dist/project.js";
+import { hasAudio } from "@twick/media-utils";
 
 // src/audio/video-audio-extractor.ts
 var VideoElementAudioExtractor = class {
@@ -28,40 +29,113 @@ var VideoElementAudioExtractor = class {
    * Extract audio by playing the video and capturing audio output
    */
   async extractAudio(startTime, duration, playbackRate = 1) {
-    const source = this.audioContext.createMediaElementSource(this.video);
-    this.destination = this.audioContext.createMediaStreamDestination();
-    source.connect(this.destination);
+    try {
+      const source = this.audioContext.createMediaElementSource(this.video);
+      this.destination = this.audioContext.createMediaStreamDestination();
+      source.connect(this.destination);
+    } catch (err) {
+      throw new Error("Video has no audio track");
+    }
     this.audioChunks = [];
-    this.mediaRecorder = new MediaRecorder(this.destination.stream, {
-      mimeType: "audio/webm"
-    });
+    let mimeType = "audio/webm";
+    if (!MediaRecorder.isTypeSupported(mimeType)) {
+      mimeType = "";
+    }
+    try {
+      this.mediaRecorder = new MediaRecorder(this.destination.stream, {
+        mimeType: mimeType || void 0
+      });
+    } catch (err) {
+      throw new Error(`Failed to create MediaRecorder: ${err}. Video may have no audio track.`);
+    }
     this.mediaRecorder.ondataavailable = (event) => {
-      if (event.data.size > 0) {
+      if (event.data && event.data.size > 0) {
         this.audioChunks.push(event.data);
       }
     };
     this.video.currentTime = startTime;
     this.video.playbackRate = playbackRate;
-    await new Promise((resolve) => {
-      this.video.addEventListener("seeked", () => resolve(), { once: true });
+    await new Promise((resolve, reject) => {
+      const seekTimeout = setTimeout(() => {
+        reject(new Error("Video seek timeout"));
+      }, 5e3);
+      this.video.addEventListener("seeked", () => {
+        clearTimeout(seekTimeout);
+        resolve();
+      }, { once: true });
+      this.video.addEventListener("error", () => {
+        clearTimeout(seekTimeout);
+        reject(new Error("Video seek error"));
+      }, { once: true });
     });
     return new Promise((resolve, reject) => {
       const recordingTimeout = setTimeout(() => {
-        reject(new Error("Audio extraction timeout"));
-      }, (duration / playbackRate + 5) * 1e3);
-      this.mediaRecorder.start();
-      this.video.play();
+        this.video.pause();
+        if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
+          this.mediaRecorder.stop();
+        }
+        reject(new Error("Audio extraction timeout - video may have no audio track"));
+      }, (duration / playbackRate + 10) * 1e3);
+      let hasData = false;
+      const dataCheckInterval = setInterval(() => {
+        if (this.audioChunks.length > 0 && this.audioChunks.some((chunk) => chunk.size > 0)) {
+          hasData = true;
+        }
+      }, 1e3);
+      this.mediaRecorder.onerror = (event) => {
+        clearInterval(dataCheckInterval);
+        clearTimeout(recordingTimeout);
+        this.video.pause();
+        reject(new Error(`MediaRecorder error: ${event}. Video may have no audio track.`));
+      };
+      try {
+        this.mediaRecorder.start(100);
+        this.video.play().catch((playErr) => {
+          clearInterval(dataCheckInterval);
+          clearTimeout(recordingTimeout);
+          reject(new Error(`Failed to play video: ${playErr}`));
+        });
+      } catch (startErr) {
+        clearInterval(dataCheckInterval);
+        clearTimeout(recordingTimeout);
+        reject(new Error(`Failed to start recording: ${startErr}`));
+      }
       setTimeout(async () => {
+        clearInterval(dataCheckInterval);
         clearTimeout(recordingTimeout);
         this.video.pause();
-        this.mediaRecorder.stop();
+        if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
+          this.mediaRecorder.stop();
+        }
+        const stopTimeout = setTimeout(() => {
+          if (this.audioChunks.length === 0 || !hasData) {
+            reject(new Error("No audio data captured - video has no audio track"));
+          }
+        }, 2e3);
         await new Promise((res) => {
-          this.mediaRecorder.addEventListener("stop", () => res(), { once: true });
+          if (this.mediaRecorder) {
+            this.mediaRecorder.addEventListener("stop", () => {
+              clearTimeout(stopTimeout);
+              res();
+            }, { once: true });
+          } else {
+            clearTimeout(stopTimeout);
+            res();
+          }
         });
         try {
+          if (this.audioChunks.length === 0 || !this.audioChunks.some((chunk) => chunk.size > 0)) {
+            throw new Error("No audio data captured - video has no audio track");
+          }
           const audioBlob = new Blob(this.audioChunks, { type: "audio/webm" });
+          if (audioBlob.size === 0) {
+            throw new Error("Audio blob is empty - video has no audio track");
+          }
           const arrayBuffer = await audioBlob.arrayBuffer();
           const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
+          if (audioBuffer.length === 0 || audioBuffer.duration === 0) {
+            throw new Error("Audio buffer is empty - video has no audio track");
+          }
           resolve(audioBuffer);
         } catch (err) {
           reject(new Error(`Failed to decode recorded audio: ${err}`));
@@ -284,25 +358,46 @@ function getFFmpegBaseURL() {
   return "/ffmpeg";
 }
 async function muxAudioVideo(options) {
+  const muxStartTime = Date.now();
   try {
+    console.log("Starting FFmpeg muxing...");
+    console.log(`  Video blob size: ${options.videoBlob.size} bytes (${(options.videoBlob.size / 1024 / 1024).toFixed(2)} MB)`);
+    console.log(`  Audio buffer size: ${options.audioBuffer.byteLength} bytes (${(options.audioBuffer.byteLength / 1024 / 1024).toFixed(2)} MB)`);
     const { FFmpeg } = await import("@ffmpeg/ffmpeg");
     const { fetchFile } = await import("@ffmpeg/util");
     const ffmpeg = new FFmpeg();
     const base = getFFmpegBaseURL();
     const coreURL = `${base}/ffmpeg-core.js`;
     const wasmURL = `${base}/ffmpeg-core.wasm`;
+    console.log(`Loading FFmpeg from ${base}`);
+    const loadStartTime = Date.now();
     await ffmpeg.load({
       coreURL,
       wasmURL
     });
+    const loadDuration = Date.now() - loadStartTime;
+    console.log(`FFmpeg loaded successfully in ${loadDuration}ms`);
+    console.log("Writing video and audio files...");
+    const writeStartTime = Date.now();
     await ffmpeg.writeFile(
       "video.mp4",
       await fetchFile(options.videoBlob)
     );
+    console.log(`  Video file written: ${options.videoBlob.size} bytes`);
     await ffmpeg.writeFile(
       "audio.wav",
       new Uint8Array(options.audioBuffer)
     );
+    const writeDuration = Date.now() - writeStartTime;
+    console.log(`  Audio file written: ${options.audioBuffer.byteLength} bytes`);
+    console.log(`Files written successfully in ${writeDuration}ms`);
+    console.log("Executing FFmpeg muxing command...");
+    const execStartTime = Date.now();
+    const ffmpegLogs = [];
+    ffmpeg.on("log", ({ message }) => {
+      ffmpegLogs.push(message);
+      console.log(`  [FFmpeg] ${message}`);
+    });
     await ffmpeg.exec([
       "-i",
       "video.mp4",
@@ -317,11 +412,34 @@ async function muxAudioVideo(options) {
       "-shortest",
       "output.mp4"
     ]);
+    const execDuration = Date.now() - execStartTime;
+    console.log(`FFmpeg muxing completed in ${execDuration}ms`);
+    const readStartTime = Date.now();
     const data = await ffmpeg.readFile("output.mp4");
+    const readDuration = Date.now() - readStartTime;
+    console.log(`Output file read successfully in ${readDuration}ms`);
     const uint8 = typeof data === "string" ? new TextEncoder().encode(data) : new Uint8Array(data);
-    return new Blob([uint8], { type: "video/mp4" });
-  } catch {
-    return options.videoBlob;
+    const result = new Blob([uint8], { type: "video/mp4" });
+    const totalDuration = Date.now() - muxStartTime;
+    console.log(`Muxing successful: ${result.size} bytes (${(result.size / 1024 / 1024).toFixed(2)} MB) in ${totalDuration}ms`);
+    console.log(`  Breakdown: load=${loadDuration}ms, write=${writeDuration}ms, exec=${execDuration}ms, read=${readDuration}ms`);
+    return result;
+  } catch (error) {
+    const totalDuration = Date.now() - muxStartTime;
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    const errorStack = error instanceof Error ? error.stack : void 0;
+    console.error("FFmpeg muxing failed:", errorMsg);
+    if (errorStack) {
+      console.error("Error stack:", errorStack);
+    }
+    console.error("Error details:", {
+      errorType: error instanceof Error ? error.constructor.name : typeof error,
+      errorMessage: errorMsg,
+      duration: `${totalDuration}ms`,
+      videoBlobSize: options.videoBlob.size,
+      audioBufferSize: options.audioBuffer.byteLength
+    });
+    throw error;
   }
 }
 
@@ -407,33 +525,99 @@ var BrowserWasmExporter = class _BrowserWasmExporter {
   }
   async generateAudio(assets, startFrame, endFrame) {
     try {
+      console.log(`Generating audio from ${assets.length} frames`);
       const processor = new BrowserAudioProcessor();
       const assetPlacements = getAssetPlacement(assets);
       if (assetPlacements.length === 0) {
+        console.log("No asset placements found");
         return null;
       }
+      console.log(`Processing ${assetPlacements.length} asset placements`);
       const processedBuffers = [];
-      for (const asset of assetPlacements) {
+      for (let i = 0; i < assetPlacements.length; i++) {
+        const asset = assetPlacements[i];
+        console.log(`[${i + 1}/${assetPlacements.length}] Processing asset: ${asset.src} (type: ${asset.type}, volume: ${asset.volume}, playbackRate: ${asset.playbackRate})`);
         if (asset.volume > 0 && asset.playbackRate > 0) {
+          const startTime = Date.now();
           try {
-            const buffer = await processor.processAudioAsset(
+            if (asset.type === "video") {
+              console.log(`  \u2192 Checking if asset has audio: ${asset.src.substring(0, 50)}...`);
+              try {
+                const assetHasAudio = await hasAudio(asset.src);
+                if (!assetHasAudio) {
+                  console.log(`  \u23ED Skipping asset (no audio detected): ${asset.src.substring(0, 50)}...`);
+                  continue;
+                }
+                console.log(`  \u2713 Asset has audio, proceeding: ${asset.src.substring(0, 50)}...`);
+              } catch (audioCheckError) {
+                const errorMsg = audioCheckError instanceof Error ? audioCheckError.message : String(audioCheckError);
+                const errorStack = audioCheckError instanceof Error ? audioCheckError.stack : void 0;
+                console.warn(`  \u26A0 Audio check failed, proceeding anyway: ${asset.src.substring(0, 50)}...`);
+                console.warn(`    Error: ${errorMsg}`);
+                if (errorStack) {
+                  console.warn(`    Stack: ${errorStack}`);
+                }
+              }
+            }
+            console.log(`  \u2192 Starting processAudioAsset for: ${asset.src}`);
+            const processPromise = processor.processAudioAsset(
               asset,
               this.settings.fps || 30,
               endFrame - startFrame
             );
+            const timeoutPromise = new Promise((_, reject) => {
+              setTimeout(() => {
+                reject(new Error(`Timeout processing audio asset after 20s - video may have no audio track`));
+              }, 2e4);
+            });
+            const buffer = await Promise.race([processPromise, timeoutPromise]);
+            const duration = Date.now() - startTime;
+            console.log(`  \u2713 Successfully processed audio asset in ${duration}ms: ${asset.src.substring(0, 50)}...`);
             processedBuffers.push(buffer);
-          } catch {
+          } catch (error) {
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            const errorStack = error instanceof Error ? error.stack : void 0;
+            const duration = Date.now() - startTime;
+            console.warn(`  \u2717 Failed to process audio asset after ${duration}ms: ${asset.src.substring(0, 50)}...`);
+            console.warn(`    Error: ${errorMsg}`);
+            if (errorStack) {
+              console.warn(`    Stack: ${errorStack}`);
+            }
+            console.warn(`    Asset details: type=${asset.type}, volume=${asset.volume}, playbackRate=${asset.playbackRate}, startFrame=${asset.startInVideo}, endFrame=${asset.endInVideo}`);
           }
+        } else {
+          console.log(`  \u23ED Skipping asset: volume=${asset.volume}, playbackRate=${asset.playbackRate}`);
         }
       }
       if (processedBuffers.length === 0) {
+        console.warn("No audio buffers were successfully processed");
+        console.warn(`  Total assets attempted: ${assetPlacements.length}`);
+        console.warn(`  Assets with volume>0 and playbackRate>0: ${assetPlacements.filter((a) => a.volume > 0 && a.playbackRate > 0).length}`);
         return null;
       }
+      console.log(`Mixing ${processedBuffers.length} audio buffers`);
+      const mixStartTime = Date.now();
       const mixedBuffer = processor.mixAudioBuffers(processedBuffers);
+      const mixDuration = Date.now() - mixStartTime;
+      console.log(`Audio mixing completed in ${mixDuration}ms`);
+      const wavStartTime = Date.now();
       const wavData = processor.audioBufferToWav(mixedBuffer);
+      const wavDuration = Date.now() - wavStartTime;
+      console.log(`WAV conversion completed in ${wavDuration}ms`);
+      console.log(`Audio generation complete: ${wavData.byteLength} bytes (${(wavData.byteLength / 1024 / 1024).toFixed(2)} MB)`);
       await processor.close();
       return wavData;
-    } catch {
+    } catch (error) {
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      const errorStack = error instanceof Error ? error.stack : void 0;
+      console.error("Audio generation error:", errorMsg);
+      if (errorStack) {
+        console.error("Error stack:", errorStack);
+      }
+      console.error("Error details:", {
+        errorType: error instanceof Error ? error.constructor.name : typeof error,
+        errorMessage: errorMsg
+      });
       return null;
     }
   }
@@ -521,6 +705,8 @@ var renderTwickVideoInBrowser = async (config) => {
         }
       });
     }
+    let hasAnyAudio = false;
+    console.log(`Found ${videoElements.length} video element(s) to check for audio`);
     if (videoElements.length > 0) {
       for (const videoEl of videoElements) {
         const src = videoEl.props?.src;
@@ -544,6 +730,23 @@ var renderTwickVideoInBrowser = async (config) => {
             reject(new Error(`Failed to load video: ${err?.message || "Unknown error"}`));
           }, { once: true });
         });
+        if (settings.includeAudio) {
+          try {
+            console.log(`Checking if video has audio: ${src.substring(0, 50)}...`);
+            const videoHasAudio = await hasAudio(src);
+            console.log(`Audio check result for ${src.substring(0, 50)}...: ${videoHasAudio ? "HAS AUDIO" : "NO AUDIO"}`);
+            if (videoHasAudio) {
+              hasAnyAudio = true;
+              console.log(`\u2713 Video has audio: ${src.substring(0, 50)}...`);
+            } else {
+              console.log(`\u2717 Video has no audio: ${src.substring(0, 50)}...`);
+            }
+          } catch (error) {
+            console.warn(`Failed to check audio for ${src.substring(0, 50)}...:`, error);
+            hasAnyAudio = true;
+            console.log(`\u26A0 Assuming video might have audio due to check error`);
+          }
+        }
       }
     }
     await renderer.playback.recalculate();
@@ -566,28 +769,103 @@ var renderTwickVideoInBrowser = async (config) => {
     }
     await exporter.stop();
     let audioData = null;
-    if (settings.includeAudio && mediaAssets.length > 0) {
-      audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
+    console.log(`Audio detection summary: hasAnyAudio=${hasAnyAudio}, includeAudio=${settings.includeAudio}, mediaAssets=${mediaAssets.length}`);
+    if (settings.includeAudio && mediaAssets.length > 0 && hasAnyAudio) {
+      console.log("Starting audio processing (audio detected in videos)");
+      if (settings.onProgress) {
+        settings.onProgress(0.98);
+      }
+      try {
+        console.log("Calling generateAudio...");
+        audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
+        console.log("generateAudio completed");
+        if (audioData) {
+          console.log(`\u2713 Audio generation successful: ${audioData.byteLength} bytes`);
+        } else {
+          console.log("\u2717 No audio data generated");
+        }
+        if (settings.onProgress) {
+          settings.onProgress(0.99);
+        }
+      } catch (audioError) {
+        const errorMsg = audioError instanceof Error ? audioError.message : String(audioError);
+        const errorStack = audioError instanceof Error ? audioError.stack : void 0;
+        console.error("\u2717 Audio generation failed, continuing without audio");
+        console.error(`  Error: ${errorMsg}`);
+        if (errorStack) {
+          console.error(`  Stack: ${errorStack}`);
+        }
+        console.error("  Context:", {
+          hasAnyAudio,
+          includeAudio: settings.includeAudio,
+          mediaAssetsCount: mediaAssets.length,
+          totalFrames
+        });
+        audioData = null;
+      }
+    } else if (settings.includeAudio && mediaAssets.length > 0 && !hasAnyAudio) {
+      console.log("\u23ED Skipping audio processing: no audio detected in videos");
+    } else {
+      console.log(`\u23ED Skipping audio processing: includeAudio=${settings.includeAudio}, mediaAssets=${mediaAssets.length}, hasAnyAudio=${hasAnyAudio}`);
     }
     let finalBlob = exporter.getVideoBlob();
     if (!finalBlob) {
       throw new Error("Failed to create video blob");
     }
+    if (finalBlob.size === 0) {
+      throw new Error("Video blob is empty. Rendering may have failed.");
+    }
     if (audioData && settings.includeAudio) {
+      console.log(`Attempting to mux audio (${audioData.byteLength} bytes) with video (${finalBlob.size} bytes)`);
       try {
-        finalBlob = await muxAudioVideo({
+        const muxedBlob = await muxAudioVideo({
           videoBlob: finalBlob,
           audioBuffer: audioData
         });
-      } catch {
-        const audioBlob = new Blob([audioData], { type: "audio/wav" });
-        const audioUrl = URL.createObjectURL(audioBlob);
-        const a = document.createElement("a");
-        a.href = audioUrl;
-        a.download = "audio.wav";
-        a.click();
-        URL.revokeObjectURL(audioUrl);
+        if (!muxedBlob || muxedBlob.size === 0) {
+          throw new Error("Muxed video blob is empty");
+        }
+        if (muxedBlob.size === finalBlob.size) {
+          console.warn("Muxed blob size unchanged - muxing may have failed silently");
+        } else {
+          console.log(`Muxing successful: ${finalBlob.size} bytes -> ${muxedBlob.size} bytes`);
+        }
+        finalBlob = muxedBlob;
+      } catch (muxError) {
+        const errorMsg = muxError instanceof Error ? muxError.message : String(muxError);
+        const errorStack = muxError instanceof Error ? muxError.stack : void 0;
+        console.error("Audio muxing failed");
+        console.error(`  Error: ${errorMsg}`);
+        if (errorStack) {
+          console.error(`  Stack: ${errorStack}`);
+        }
+        console.error("  Context:", {
+          videoBlobSize: finalBlob.size,
+          audioDataSize: audioData?.byteLength || 0
+        });
+        if (settings.downloadAudioSeparately && audioData) {
+          const audioBlob = new Blob([audioData], { type: "audio/wav" });
+          const audioUrl = URL.createObjectURL(audioBlob);
+          const a = document.createElement("a");
+          a.href = audioUrl;
+          a.download = "audio.wav";
+          a.click();
+          URL.revokeObjectURL(audioUrl);
+        }
+        console.warn("Continuing with video without audio due to muxing failure");
+        finalBlob = exporter.getVideoBlob();
+        if (!finalBlob || finalBlob.size === 0) {
+          throw new Error("Video blob is invalid after muxing failure");
+        }
       }
+    } else if (settings.includeAudio && !audioData) {
+      console.warn("Audio processing was enabled but no audio data was generated");
+    }
+    if (!finalBlob || finalBlob.size === 0) {
+      throw new Error("Final video blob is empty or invalid");
+    }
+    if (settings.onProgress) {
+      settings.onProgress(1);
     }
     if (settings.onComplete) {
       settings.onComplete(finalBlob);