@twick/browser-render 0.15.6 → 0.15.8

package/dist/index.mjs

@@ -0,0 +1,708 @@
+// src/browser-renderer.ts
+import { Renderer, Vector2 } from "@twick/core";
+import defaultProject from "@twick/visualizer/dist/project.js";
+
+// src/audio/video-audio-extractor.ts
+var VideoElementAudioExtractor = class {
+  audioContext;
+  video;
+  destination = null;
+  mediaRecorder = null;
+  audioChunks = [];
+  constructor(videoSrc, sampleRate = 48e3) {
+    this.audioContext = new AudioContext({ sampleRate });
+    this.video = document.createElement("video");
+    this.video.crossOrigin = "anonymous";
+    this.video.src = videoSrc;
+    this.video.muted = true;
+  }
+  async initialize() {
+    return new Promise((resolve, reject) => {
+      this.video.addEventListener("loadedmetadata", () => resolve(), { once: true });
+      this.video.addEventListener("error", (e) => {
+        reject(new Error(`Failed to load video for audio extraction: ${e}`));
+      }, { once: true });
+    });
+  }
+  /**
+   * Extract audio by playing the video and capturing audio output
+   */
+  async extractAudio(startTime, duration, playbackRate = 1) {
+    const source = this.audioContext.createMediaElementSource(this.video);
+    this.destination = this.audioContext.createMediaStreamDestination();
+    source.connect(this.destination);
+    this.audioChunks = [];
+    this.mediaRecorder = new MediaRecorder(this.destination.stream, {
+      mimeType: "audio/webm"
+    });
+    this.mediaRecorder.ondataavailable = (event) => {
+      if (event.data.size > 0) {
+        this.audioChunks.push(event.data);
+      }
+    };
+    this.video.currentTime = startTime;
+    this.video.playbackRate = playbackRate;
+    await new Promise((resolve) => {
+      this.video.addEventListener("seeked", () => resolve(), { once: true });
+    });
+    return new Promise((resolve, reject) => {
+      const recordingTimeout = setTimeout(() => {
+        reject(new Error("Audio extraction timeout"));
+      }, (duration / playbackRate + 5) * 1e3);
+      this.mediaRecorder.start();
+      this.video.play();
+      setTimeout(async () => {
+        clearTimeout(recordingTimeout);
+        this.video.pause();
+        this.mediaRecorder.stop();
+        await new Promise((res) => {
+          this.mediaRecorder.addEventListener("stop", () => res(), { once: true });
+        });
+        try {
+          const audioBlob = new Blob(this.audioChunks, { type: "audio/webm" });
+          const arrayBuffer = await audioBlob.arrayBuffer();
+          const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
+          resolve(audioBuffer);
+        } catch (err) {
+          reject(new Error(`Failed to decode recorded audio: ${err}`));
+        }
+      }, duration / playbackRate * 1e3);
+    });
+  }
+  async close() {
+    if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
+      this.mediaRecorder.stop();
+    }
+    this.video.pause();
+    this.video.src = "";
+    if (this.audioContext.state !== "closed") {
+      await this.audioContext.close();
+    }
+  }
+};
+async function extractAudioFromVideo(videoSrc, startTime, duration, playbackRate = 1, sampleRate = 48e3) {
+  const extractor = new VideoElementAudioExtractor(videoSrc, sampleRate);
+  try {
+    await extractor.initialize();
+    const audioBuffer = await extractor.extractAudio(startTime, duration, playbackRate);
+    return audioBuffer;
+  } finally {
+    await extractor.close();
+  }
+}
+
+// src/audio/audio-processor.ts
+function getAssetPlacement(frames) {
+  const assets = [];
+  const assetTimeMap = /* @__PURE__ */ new Map();
+  for (let frame = 0; frame < frames.length; frame++) {
+    for (const asset of frames[frame]) {
+      if (!assetTimeMap.has(asset.key)) {
+        assetTimeMap.set(asset.key, {
+          start: asset.currentTime,
+          end: asset.currentTime
+        });
+        assets.push({
+          key: asset.key,
+          src: asset.src,
+          type: asset.type,
+          startInVideo: frame,
+          endInVideo: frame,
+          duration: 0,
+          durationInSeconds: 0,
+          playbackRate: asset.playbackRate,
+          volume: asset.volume,
+          trimLeftInSeconds: asset.currentTime
+        });
+      } else {
+        const timeInfo = assetTimeMap.get(asset.key);
+        if (timeInfo) {
+          timeInfo.end = asset.currentTime;
+        }
+        const existingAsset = assets.find((a) => a.key === asset.key);
+        if (existingAsset) {
+          existingAsset.endInVideo = frame;
+        }
+      }
+    }
+  }
+  assets.forEach((asset) => {
+    const timeInfo = assetTimeMap.get(asset.key);
+    if (timeInfo) {
+      asset.durationInSeconds = (timeInfo.end - timeInfo.start) / asset.playbackRate;
+    }
+    asset.duration = asset.endInVideo - asset.startInVideo + 1;
+  });
+  return assets;
+}
+var BrowserAudioProcessor = class {
+  constructor(sampleRate = 48e3) {
+    this.sampleRate = sampleRate;
+    this.audioContext = new AudioContext({ sampleRate });
+  }
+  audioContext;
+  /**
+   * Fetch and decode audio from a media source
+   * Falls back to video element extraction if decodeAudioData fails
+   */
+  async fetchAndDecodeAudio(src) {
+    try {
+      const response = await fetch(src);
+      const arrayBuffer = await response.arrayBuffer();
+      return await this.audioContext.decodeAudioData(arrayBuffer);
+    } catch (err) {
+      try {
+        return await extractAudioFromVideo(
+          src,
+          0,
+          999999,
+          1,
+          this.sampleRate
+        );
+      } catch (fallbackErr) {
+        throw new Error(`Failed to extract audio: ${err}. Fallback also failed: ${fallbackErr}`);
+      }
+    }
+  }
+  /**
+   * Process audio asset with playback rate, volume, and timing
+   */
+  async processAudioAsset(asset, fps, totalFrames) {
+    const audioBuffer = await this.fetchAndDecodeAudio(asset.src);
+    const duration = totalFrames / fps;
+    const outputLength = Math.ceil(duration * this.sampleRate);
+    const outputBuffer = this.audioContext.createBuffer(
+      2,
+      // stereo
+      outputLength,
+      this.sampleRate
+    );
+    const startTime = asset.startInVideo / fps;
+    const trimLeft = asset.trimLeftInSeconds / asset.playbackRate;
+    const trimRight = trimLeft + asset.durationInSeconds;
+    for (let channel = 0; channel < 2; channel++) {
+      const inputData = audioBuffer.getChannelData(Math.min(channel, audioBuffer.numberOfChannels - 1));
+      const outputData = outputBuffer.getChannelData(channel);
+      const startSample = Math.floor(startTime * this.sampleRate);
+      const trimLeftSample = Math.floor(trimLeft * this.sampleRate);
+      const trimRightSample = Math.floor(trimRight * this.sampleRate);
+      for (let i = 0; i < outputData.length; i++) {
+        const outputTime = i / this.sampleRate;
+        const assetTime = outputTime - startTime;
+        if (assetTime < 0 || assetTime >= asset.durationInSeconds) {
+          outputData[i] = 0;
+        } else {
+          const inputSample = Math.floor(trimLeftSample + assetTime * asset.playbackRate * this.sampleRate);
+          if (inputSample >= 0 && inputSample < inputData.length) {
+            outputData[i] = inputData[inputSample] * asset.volume;
+          } else {
+            outputData[i] = 0;
+          }
+        }
+      }
+    }
+    return outputBuffer;
+  }
+  /**
+   * Mix multiple audio buffers
+   */
+  mixAudioBuffers(buffers) {
+    if (buffers.length === 0) {
+      return this.audioContext.createBuffer(2, 1, this.sampleRate);
+    }
+    const maxLength = Math.max(...buffers.map((b) => b.length));
+    const mixedBuffer = this.audioContext.createBuffer(2, maxLength, this.sampleRate);
+    for (let channel = 0; channel < 2; channel++) {
+      const mixedData = mixedBuffer.getChannelData(channel);
+      buffers.forEach((buffer) => {
+        const channelData = buffer.getChannelData(Math.min(channel, buffer.numberOfChannels - 1));
+        for (let i = 0; i < channelData.length; i++) {
+          mixedData[i] = (mixedData[i] || 0) + channelData[i] / buffers.length;
+        }
+      });
+    }
+    return mixedBuffer;
+  }
+  /**
+   * Convert AudioBuffer to WAV format
+   */
+  audioBufferToWav(buffer) {
+    const numberOfChannels = buffer.numberOfChannels;
+    const sampleRate = buffer.sampleRate;
+    const format = 1;
+    const bitDepth = 16;
+    const bytesPerSample = bitDepth / 8;
+    const blockAlign = numberOfChannels * bytesPerSample;
+    const data = new Float32Array(buffer.length * numberOfChannels);
+    for (let channel = 0; channel < numberOfChannels; channel++) {
+      const channelData = buffer.getChannelData(channel);
+      for (let i = 0; i < buffer.length; i++) {
+        data[i * numberOfChannels + channel] = channelData[i];
+      }
+    }
+    const dataLength = data.length * bytesPerSample;
+    const headerLength = 44;
+    const wav = new ArrayBuffer(headerLength + dataLength);
+    const view = new DataView(wav);
+    const writeString = (offset2, string) => {
+      for (let i = 0; i < string.length; i++) {
+        view.setUint8(offset2 + i, string.charCodeAt(i));
+      }
+    };
+    writeString(0, "RIFF");
+    view.setUint32(4, 36 + dataLength, true);
+    writeString(8, "WAVE");
+    writeString(12, "fmt ");
+    view.setUint32(16, 16, true);
+    view.setUint16(20, format, true);
+    view.setUint16(22, numberOfChannels, true);
+    view.setUint32(24, sampleRate, true);
+    view.setUint32(28, sampleRate * blockAlign, true);
+    view.setUint16(32, blockAlign, true);
+    view.setUint16(34, bitDepth, true);
+    writeString(36, "data");
+    view.setUint32(40, dataLength, true);
+    const volume = 0.8;
+    let offset = 44;
+    for (let i = 0; i < data.length; i++) {
+      const sample = Math.max(-1, Math.min(1, data[i]));
+      view.setInt16(offset, sample < 0 ? sample * 32768 : sample * 32767, true);
+      offset += 2;
+    }
+    return wav;
+  }
+  async close() {
+    await this.audioContext.close();
+  }
+};
+
+// src/audio/audio-video-muxer.ts
+function getFFmpegBaseURL() {
+  if (typeof window !== "undefined") {
+    return `${window.location.origin}/ffmpeg`;
+  }
+  return "/ffmpeg";
+}
+async function muxAudioVideo(options) {
+  try {
+    const { FFmpeg } = await import("@ffmpeg/ffmpeg");
+    const { fetchFile } = await import("@ffmpeg/util");
+    const ffmpeg = new FFmpeg();
+    const base = getFFmpegBaseURL();
+    const coreURL = `${base}/ffmpeg-core.js`;
+    const wasmURL = `${base}/ffmpeg-core.wasm`;
+    await ffmpeg.load({
+      coreURL,
+      wasmURL
+    });
+    await ffmpeg.writeFile(
+      "video.mp4",
+      await fetchFile(options.videoBlob)
+    );
+    await ffmpeg.writeFile(
+      "audio.wav",
+      new Uint8Array(options.audioBuffer)
+    );
+    await ffmpeg.exec([
+      "-i",
+      "video.mp4",
+      "-i",
+      "audio.wav",
+      "-c:v",
+      "copy",
+      "-c:a",
+      "aac",
+      "-b:a",
+      "192k",
+      "-shortest",
+      "output.mp4"
+    ]);
+    const data = await ffmpeg.readFile("output.mp4");
+    const uint8 = typeof data === "string" ? new TextEncoder().encode(data) : new Uint8Array(data);
+    return new Blob([uint8], { type: "video/mp4" });
+  } catch {
+    return options.videoBlob;
+  }
+}
+
+// src/browser-renderer.ts
+var BrowserWasmExporter = class _BrowserWasmExporter {
+  constructor(settings) {
+    this.settings = settings;
+    this.fps = settings.fps || 30;
+  }
+  static id = "@twick/core/wasm";
+  static displayName = "Browser Video (Wasm)";
+  encoder;
+  videoBlob = null;
+  onProgressCallback;
+  currentFrame = 0;
+  fps = 30;
+  static async create(settings) {
+    return new _BrowserWasmExporter(settings);
+  }
+  async start() {
+    try {
+      const loadMp4Module = (await import("mp4-wasm")).default;
+      const possiblePaths = [
+        // Vite dev server virtual path
+        "/@mp4-wasm",
+        // Common bundled asset paths (Vite uses hashed names)
+        "/assets/mp4-wasm.wasm",
+        "/assets/mp4-YBRi_559.wasm",
+        // Known Vite hash
+        "/mp4-wasm.wasm",
+        // Node modules path (for dev)
+        "/node_modules/mp4-wasm/dist/mp4-wasm.wasm"
+      ];
+      let buffer = null;
+      let successPath = "";
+      for (const path of possiblePaths) {
+        try {
+          const resp = await fetch(path);
+          if (resp.ok) {
+            const contentType = resp.headers.get("content-type");
+            if (contentType && contentType.includes("html")) {
+              continue;
+            }
+            buffer = await resp.arrayBuffer();
+            successPath = path;
+            break;
+          }
+        } catch (e) {
+          continue;
+        }
+      }
+      if (!buffer) {
+        throw new Error(
+          "Could not load WASM file from any location. Please copy mp4-wasm.wasm to your public directory or configure Vite to serve it."
+        );
+      }
+      const mp4 = await loadMp4Module({ wasmBinary: buffer });
+      this.encoder = mp4.createWebCodecsEncoder({
+        width: this.settings.size.x,
+        height: this.settings.size.y,
+        fps: this.fps
+      });
+    } catch (error) {
+      throw error;
+    }
+  }
+  async handleFrame(canvas, frameNumber) {
+    const frameIndex = frameNumber !== void 0 ? frameNumber : this.currentFrame;
+    const timestampMicroseconds = Math.round(frameIndex / this.fps * 1e6);
+    const frame = new VideoFrame(canvas, {
+      timestamp: timestampMicroseconds,
+      duration: Math.round(1 / this.fps * 1e6)
+    });
+    await this.encoder.addFrame(frame);
+    frame.close();
+    if (frameNumber === void 0) {
+      this.currentFrame++;
+    }
+  }
+  async stop() {
+    const buf = await this.encoder.end();
+    this.videoBlob = new Blob([buf], { type: "video/mp4" });
+  }
+  async generateAudio(assets, startFrame, endFrame) {
+    try {
+      const processor = new BrowserAudioProcessor();
+      const assetPlacements = getAssetPlacement(assets);
+      if (assetPlacements.length === 0) {
+        return null;
+      }
+      const processedBuffers = [];
+      for (const asset of assetPlacements) {
+        if (asset.volume > 0 && asset.playbackRate > 0) {
+          try {
+            const buffer = await processor.processAudioAsset(
+              asset,
+              this.settings.fps || 30,
+              endFrame - startFrame
+            );
+            processedBuffers.push(buffer);
+          } catch {
+          }
+        }
+      }
+      if (processedBuffers.length === 0) {
+        return null;
+      }
+      const mixedBuffer = processor.mixAudioBuffers(processedBuffers);
+      const wavData = processor.audioBufferToWav(mixedBuffer);
+      await processor.close();
+      return wavData;
+    } catch {
+      return null;
+    }
+  }
+  async mergeMedia() {
+  }
+  async downloadVideos(assets) {
+  }
+  getVideoBlob() {
+    return this.videoBlob;
+  }
+  setProgressCallback(callback) {
+    this.onProgressCallback = callback;
+  }
+};
+var renderTwickVideoInBrowser = async (config) => {
+  const originalVideoPlay = HTMLVideoElement.prototype.play;
+  const originalAudioPlay = HTMLAudioElement.prototype.play;
+  const originalCreateElement = document.createElement.bind(document);
+  HTMLVideoElement.prototype.play = function() {
+    this.muted = true;
+    this.volume = 0;
+    return originalVideoPlay.call(this);
+  };
+  HTMLAudioElement.prototype.play = function() {
+    this.muted = true;
+    this.volume = 0;
+    return originalAudioPlay.call(this);
+  };
+  document.createElement = function(tagName, options) {
+    const element = originalCreateElement(tagName, options);
+    if (tagName.toLowerCase() === "video" || tagName.toLowerCase() === "audio") {
+      element.muted = true;
+      element.volume = 0;
+    }
+    return element;
+  };
+  try {
+    const { projectFile, variables, settings = {} } = config;
+    if (!variables || !variables.input) {
+      throw new Error('Invalid configuration. "variables.input" is required.');
+    }
+    const width = settings.width || variables.input.properties?.width || 1920;
+    const height = settings.height || variables.input.properties?.height || 1080;
+    const fps = settings.fps || variables.input.properties?.fps || 30;
+    const project = !projectFile ? defaultProject : projectFile;
+    project.variables = variables;
+    const renderSettings = {
+      name: "browser-render",
+      exporter: {
+        name: "@twick/core/wasm"
+      },
+      size: new Vector2(width, height),
+      resolutionScale: 1,
+      colorSpace: "srgb",
+      fps,
+      range: settings.range || [0, Infinity],
+      background: variables.input.backgroundColor || "#000000",
+      ...settings.quality && {
+        quality: settings.quality
+      }
+    };
+    const renderer = new Renderer(project);
+    const exporter = await BrowserWasmExporter.create(renderSettings);
+    await exporter.start();
+    if (settings.onProgress) {
+      exporter.setProgressCallback(settings.onProgress);
+    }
+    await renderer["reloadScenes"](renderSettings);
+    renderer.stage.configure(renderSettings);
+    renderer.playback.fps = renderSettings.fps;
+    renderer.playback.state = 1;
+    const totalFrames = await renderer.getNumberOfFrames(renderSettings);
+    if (totalFrames === 0 || !isFinite(totalFrames)) {
+      throw new Error(
+        "Cannot render: Video has zero duration. Please ensure your project has valid content with non-zero duration. Check that all video elements have valid sources and are properly loaded."
+      );
+    }
+    const videoElements = [];
+    if (variables.input.tracks) {
+      variables.input.tracks.forEach((track) => {
+        if (track.elements) {
+          track.elements.forEach((el) => {
+            if (el.type === "video") videoElements.push(el);
+          });
+        }
+      });
+    }
+    if (videoElements.length > 0) {
+      for (const videoEl of videoElements) {
+        const src = videoEl.props?.src;
+        if (!src || src === "undefined") continue;
+        const preloadVideo = document.createElement("video");
+        preloadVideo.crossOrigin = "anonymous";
+        preloadVideo.preload = "metadata";
+        preloadVideo.src = src;
+        await new Promise((resolve, reject) => {
+          const timeout = setTimeout(
+            () => reject(new Error(`Timeout loading video metadata: ${src.substring(0, 80)}`)),
+            3e4
+          );
+          preloadVideo.addEventListener("loadedmetadata", () => {
+            clearTimeout(timeout);
+            resolve();
+          }, { once: true });
+          preloadVideo.addEventListener("error", () => {
+            clearTimeout(timeout);
+            const err = preloadVideo.error;
+            reject(new Error(`Failed to load video: ${err?.message || "Unknown error"}`));
+          }, { once: true });
+        });
+      }
+    }
+    await renderer.playback.recalculate();
+    await renderer.playback.reset();
+    await renderer.playback.seek(0);
+    const mediaAssets = [];
+    for (let frame = 0; frame < totalFrames; frame++) {
+      if (frame > 0) {
+        await renderer.playback.progress();
+      }
+      await renderer.stage.render(
+        renderer.playback.currentScene,
+        renderer.playback.previousScene
+      );
+      const currentAssets = renderer.playback.currentScene.getMediaAssets?.() || [];
+      mediaAssets.push(currentAssets);
+      const canvas = renderer.stage.finalBuffer;
+      await exporter.handleFrame(canvas, frame);
+      if (settings.onProgress) settings.onProgress(frame / totalFrames);
+    }
+    await exporter.stop();
+    let audioData = null;
+    if (settings.includeAudio && mediaAssets.length > 0) {
+      audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
+    }
+    let finalBlob = exporter.getVideoBlob();
+    if (!finalBlob) {
+      throw new Error("Failed to create video blob");
+    }
+    if (audioData && settings.includeAudio) {
+      try {
+        finalBlob = await muxAudioVideo({
+          videoBlob: finalBlob,
+          audioBuffer: audioData
+        });
+      } catch {
+        const audioBlob = new Blob([audioData], { type: "audio/wav" });
+        const audioUrl = URL.createObjectURL(audioBlob);
+        const a = document.createElement("a");
+        a.href = audioUrl;
+        a.download = "audio.wav";
+        a.click();
+        URL.revokeObjectURL(audioUrl);
+      }
+    }
+    if (settings.onComplete) {
+      settings.onComplete(finalBlob);
+    }
+    return finalBlob;
+  } catch (error) {
+    if (config.settings?.onError) {
+      config.settings.onError(error);
+    }
+    throw error;
+  } finally {
+    HTMLVideoElement.prototype.play = originalVideoPlay;
+    HTMLAudioElement.prototype.play = originalAudioPlay;
+    document.createElement = originalCreateElement;
+  }
+};
+var downloadVideoBlob = (videoBlob, filename = "video.mp4") => {
+  const url = URL.createObjectURL(videoBlob);
+  const a = document.createElement("a");
+  a.href = url;
+  a.download = filename;
+  a.style.display = "none";
+  document.body.appendChild(a);
+  a.click();
+  document.body.removeChild(a);
+  setTimeout(() => URL.revokeObjectURL(url), 1e3);
+};
+
+// src/hooks/use-browser-renderer.ts
+import { useState, useCallback } from "react";
+var useBrowserRenderer = (options = {}) => {
+  const [progress, setProgress] = useState(0);
+  const [isRendering, setIsRendering] = useState(false);
+  const [error, setError] = useState(null);
+  const [videoBlob, setVideoBlob] = useState(null);
+  const reset = useCallback(() => {
+    setProgress(0);
+    setIsRendering(false);
+    setError(null);
+    setVideoBlob(null);
+  }, []);
+  const download = useCallback((filename) => {
+    if (!videoBlob) {
+      setError(new Error("No video available to download. Please render the video first."));
+      return;
+    }
+    try {
+      downloadVideoBlob(videoBlob, filename || options.downloadFilename || "video.mp4");
+    } catch (err) {
+      setError(err instanceof Error ? err : new Error("Failed to download video"));
+    }
+  }, [videoBlob, options.downloadFilename]);
+  const render = useCallback(async (variables) => {
+    reset();
+    setIsRendering(true);
+    try {
+      const { projectFile, width, height, fps, quality, range, includeAudio, downloadAudioSeparately, onAudioReady, autoDownload, downloadFilename, ...restOptions } = options;
+      const blob = await renderTwickVideoInBrowser({
+        projectFile,
+        variables,
+        settings: {
+          width,
+          height,
+          includeAudio,
+          downloadAudioSeparately,
+          onAudioReady,
+          fps,
+          quality,
+          range,
+          ...restOptions,
+          onProgress: (p) => {
+            setProgress(p);
+          },
+          onComplete: (blob2) => {
+            setVideoBlob(blob2);
+            if (autoDownload) {
+              try {
+                downloadVideoBlob(blob2, downloadFilename || "video.mp4");
+              } catch (downloadErr) {
+                setError(downloadErr instanceof Error ? downloadErr : new Error("Failed to auto-download video"));
+              }
+            }
+          },
+          onError: (err) => {
+            setError(err);
+          }
+        }
+      });
+      if (!blob) {
+        throw new Error("Rendering failed: No video blob was generated");
+      }
+      setVideoBlob(blob);
+      setProgress(1);
+      return blob;
+    } catch (err) {
+      setError(err instanceof Error ? err : new Error(String(err)));
+      return null;
+    } finally {
+      setIsRendering(false);
+    }
+  }, [options, reset]);
+  return {
+    render,
+    progress,
+    isRendering,
+    error,
+    videoBlob,
+    download,
+    reset
+  };
+};
+export {
+  renderTwickVideoInBrowser as default,
+  downloadVideoBlob,
+  renderTwickVideoInBrowser,
+  useBrowserRenderer
+};
+//# sourceMappingURL=index.mjs.map