@twick/browser-render 0.15.6 → 0.15.7

package/package.json

@@ -1,30 +1,35 @@
 {
   "name": "@twick/browser-render",
-  "version": "0.15.6",
+  "version": "0.15.7",
   "license": "https://github.com/ncounterspecialist/twick/blob/main/LICENSE.md",
   "description": "Browser-native video rendering for Twick using WebCodecs API",
-  "main": "./dist/index.cjs",
+  "main": "./dist/index.js",
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.mjs",
-      "require": "./dist/index.cjs"
+      "require": "./dist/index.js"
     }
   },
+  "files": [
+    "dist",
+    "public"
+  ],
   "scripts": {
-    "build": "tsup",
+    "build": "tsup && node scripts/copy-wasm.js",
     "dev": "tsup --watch",
     "test:browser": "tsx src/test-browser-render.ts",
-    "clean": "rimraf dist"
+    "clean": "rimraf dist",
+    "docs": "typedoc"
   },
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
-    "@twick/core": "^0.15.6",
-    "@twick/visualizer": "0.15.6",
+    "@twick/core": "^0.15.7",
+    "@twick/visualizer": "0.15.7",
     "mp4-wasm": "^1.0.6",
     "mp4box": "^0.5.2",
     "@ffmpeg/ffmpeg": "^0.12.10",
@@ -45,7 +50,9 @@
     "rimraf": "^5.0.5",
     "tsup": "^8.0.0",
     "tsx": "^4.7.0",
-    "typescript": "5.4.2"
+    "typescript": "5.4.2",
+    "typedoc": "^0.25.8",
+    "typedoc-plugin-markdown": "^3.17.1"
   },
   "engines": {
     "node": ">=20.0.0"

package/AUDIO_IMPLEMENTATION.md

@@ -1,217 +0,0 @@
-# Audio Implementation Guide
-
-## Overview
-
-Browser-based audio processing for Twick video rendering, matching the server's FFmpeg implementation.
-
-## Architecture
-
-### 1. **Audio Processor** (`src/audio/audio-processor.ts`)
-- Mirrors server's `generate-audio.ts` logic
-- Uses Web Audio API instead of FFmpeg
-- Handles:
-  - Asset tracking across frames
-  - Audio extraction from video/audio elements
-  - Playback rate adjustment
-  - Volume control
-  - Audio trimming
-  - Multi-track mixing
-
-### 2. **Service Worker** (`public/audio-worker.js`)
-- Caches media assets for offline processing
-- Handles background audio extraction
-- Reduces network requests during rendering
-
-### 3. **Audio/Video Muxer** (`src/audio/audio-video-muxer.ts`)
-- Combines video and audio tracks
-- Two approaches:
-  - **mp4box.js**: Lightweight, browser-native
-  - **FFmpeg.wasm**: More robust, ~30MB bundle
-
-## Implementation Steps
-
-### Step 1: Install Dependencies
-
-```bash
-# For mp4box.js approach (recommended)
-npm install mp4box
-
-# OR for FFmpeg.wasm approach (more features, larger)
-npm install @ffmpeg/ffmpeg @ffmpeg/util
-```
-
-### Step 2: Register Service Worker
-
-```typescript
-// In your app's initialization
-if ('serviceWorker' in navigator) {
-  navigator.serviceWorker.register('/audio-worker.js')
-    .then(reg => console.log('Audio worker registered'))
-    .catch(err => console.error('Audio worker failed:', err));
-}
-```
-
-### Step 3: Enable Audio in Browser Renderer
-
-Update `browser-renderer.ts`:
-
-```typescript
-import { BrowserAudioProcessor, getAssetPlacement } from './audio/audio-processor';
-import { muxAudioVideo } from './audio/audio-video-muxer';
-
-// In BrowserWasmExporter.generateAudio():
-public async generateAudio(
-  assets: any[][],
-  startFrame: number,
-  endFrame: number,
-): Promise<ArrayBuffer | null> {
-  const processor = new BrowserAudioProcessor();
-  const assetPlacements = getAssetPlacement(assets);
-  
-  const processedBuffers: AudioBuffer[] = [];
-  for (const asset of assetPlacements) {
-    if (asset.volume > 0 && asset.playbackRate > 0) {
-      const buffer = await processor.processAudioAsset(
-        asset,
-        this.settings.fps || 30,
-        endFrame - startFrame
-      );
-      processedBuffers.push(buffer);
-    }
-  }
-  
-  const mixedBuffer = processor.mixAudioBuffers(processedBuffers);
-  const wavData = processor.audioBufferToWav(mixedBuffer);
-  
-  await processor.close();
-  return wavData;
-}
-```
-
-### Step 4: Collect Audio Assets During Rendering
-
-In `renderTwickVideoInBrowser()`:
-
-```typescript
-// Track media assets for each frame
-const mediaAssets: AssetInfo[][] = [];
-
-for (let frame = 0; frame < totalFrames; frame++) {
-  // ... existing rendering code ...
-  
-  // Collect media assets from current scene
-  const currentAssets = (renderer as any).playback.currentScene.getMediaAssets?.() || [];
-  mediaAssets.push(currentAssets);
-}
-
-// Generate audio after video rendering
-const audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
-
-// Mux audio and video
-if (audioData) {
-  const finalBlob = await muxAudioVideo({
-    videoBlob,
-    audioBuffer: audioData,
-    fps,
-    width,
-    height
-  });
-  return finalBlob;
-}
-```
-
-## API Parity with Server
-
-| Feature | Server (FFmpeg) | Browser (Web Audio) | Status |
-|---------|-----------------|---------------------|--------|
-| Asset Tracking | `getAssetPlacement()` | `getAssetPlacement()` | ✅ Ready |
-| Audio Extraction | FFmpeg decode | `decodeAudioData()` | ✅ Ready |
-| Playback Rate | `atempo` filter | Sample interpolation | ✅ Ready |
-| Volume | `volume` filter | Gain multiplication | ✅ Ready |
-| Trimming | `atrim` filter | Sample slicing | ✅ Ready |
-| Mixing | `amix` filter | Buffer mixing | ✅ Ready |
-| WAV Encoding | FFmpeg encode | Manual WAV encoding | ✅ Ready |
-| Muxing | FFmpeg merge | mp4box.js / FFmpeg.wasm | ⚠️ Needs library |
-
-## Performance Considerations
-
-### Memory Usage
-- Web Audio API decodes entire audio files into memory
-- Large video files can cause memory issues
-- Consider chunked processing for long videos
-
-### Processing Time
-- Audio processing adds 20-50% to render time
-- Service worker caching helps with repeated renders
-- Consider showing separate progress for video/audio
-
-### Browser Limits
-- Chrome: ~2GB audio buffer limit
-- Safari: Stricter memory limits
-- Firefox: Better memory management but slower
-
-## Example Usage
-
-```typescript
-import { useBrowserRenderer } from '@twick/browser-render';
-
-function VideoRenderer() {
-  const { render, progress } = useBrowserRenderer({
-    width: 1920,
-    height: 1080,
-    fps: 30,
-    includeAudio: true, // Enable audio processing
-  });
-
-  const handleRender = async () => {
-    const videoBlob = await render({
-      input: {
-        properties: { width: 1920, height: 1080, fps: 30 },
-        tracks: [
-          {
-            type: 'element',
-            elements: [
-              {
-                type: 'video',
-                src: 'https://example.com/video.mp4',
-                // Audio will be automatically extracted and included
-              }
-            ]
-          }
-        ]
-      }
-    });
-  };
-
-  return <button onClick={handleRender}>Render with Audio</button>;
-}
-```
-
-## Troubleshooting
-
-### No Audio in Output
-1. Check if `includeAudio: true` is set
-2. Verify service worker is registered
-3. Check browser console for Web Audio API errors
-4. Ensure video sources have audio tracks
-
-### Memory Errors
-1. Reduce video quality/resolution
-2. Process shorter segments
-3. Clear service worker cache
-4. Use FFmpeg.wasm with streaming
-
-### Performance Issues
-1. Use service worker caching
-2. Reduce number of audio tracks
-3. Lower audio sample rate (default: 48kHz)
-4. Consider server-side rendering for production
-
-## Future Enhancements
-
-- [ ] Streaming audio processing (chunks)
-- [ ] Web Workers for parallel processing
-- [ ] Real-time audio preview
-- [ ] Audio effects (reverb, EQ, etc.)
-- [ ] WASM-based audio processing for better performance
-- [ ] Support for more audio formats

package/package.json.bak

@@ -1,53 +0,0 @@
-{
-  "name": "@twick/browser-render",
-  "version": "0.15.6",
-  "license": "https://github.com/ncounterspecialist/twick/blob/main/LICENSE.md",
-  "description": "Browser-native video rendering for Twick using WebCodecs API",
-  "main": "./dist/index.cjs",
-  "module": "./dist/index.mjs",
-  "types": "./dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.mjs",
-      "require": "./dist/index.cjs"
-    }
-  },
-  "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
-    "test:browser": "tsx src/test-browser-render.ts",
-    "clean": "rimraf dist"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "dependencies": {
-    "@twick/core": "^0.15.6",
-    "@twick/visualizer": "0.15.6",
-    "mp4-wasm": "^1.0.6",
-    "mp4box": "^0.5.2",
-    "@ffmpeg/ffmpeg": "^0.12.10",
-    "@ffmpeg/util": "^0.12.1",
-    "@ffmpeg/core": "^0.12.6"
-  },
-  "peerDependencies": {
-    "react": ">=17.0.0"
-  },
-  "peerDependenciesMeta": {
-    "react": {
-      "optional": true
-    }
-  },
-  "devDependencies": {
-    "@types/node": "^20.10.0",
-    "@types/react": "^18.2.0",
-    "rimraf": "^5.0.5",
-    "tsup": "^8.0.0",
-    "tsx": "^4.7.0",
-    "typescript": "5.4.2"
-  },
-  "engines": {
-    "node": ">=20.0.0"
-  }
-}

package/src/audio/audio-processor.ts

@@ -1,239 +0,0 @@
-/**
- * Browser-based audio processing using Web Audio API
- * Mirrors the server's FFmpeg audio generation logic
- */
-
-export interface MediaAsset {
-  key: string;
-  src: string;
-  type: 'video' | 'audio';
-  startInVideo: number;
-  endInVideo: number;
-  duration: number;
-  playbackRate: number;
-  volume: number;
-  trimLeftInSeconds: number;
-  durationInSeconds: number;
-}
-
-export interface AssetInfo {
-  key: string;
-  src: string;
-  type: 'video' | 'audio';
-  currentTime: number;
-  playbackRate: number;
-  volume: number;
-}
-
-/**
- * Get asset placement from frames (similar to server's getAssetPlacement)
- */
-export function getAssetPlacement(frames: AssetInfo[][]): MediaAsset[] {
-  const assets: MediaAsset[] = [];
-  const assetTimeMap = new Map<string, { start: number; end: number }>();
-
-  for (let frame = 0; frame < frames.length; frame++) {
-    for (const asset of frames[frame]) {
-      if (!assetTimeMap.has(asset.key)) {
-        assetTimeMap.set(asset.key, {
-          start: asset.currentTime,
-          end: asset.currentTime,
-        });
-        assets.push({
-          key: asset.key,
-          src: asset.src,
-          type: asset.type,
-          startInVideo: frame,
-          endInVideo: frame,
-          duration: 0,
-          durationInSeconds: 0,
-          playbackRate: asset.playbackRate,
-          volume: asset.volume,
-          trimLeftInSeconds: asset.currentTime,
-        });
-      } else {
-        const timeInfo = assetTimeMap.get(asset.key);
-        if (timeInfo) {
-          timeInfo.end = asset.currentTime;
-        }
-        const existingAsset = assets.find(a => a.key === asset.key);
-        if (existingAsset) {
-          existingAsset.endInVideo = frame;
-        }
-      }
-    }
-  }
-
-  // Calculate durations
-  assets.forEach(asset => {
-    const timeInfo = assetTimeMap.get(asset.key);
-    if (timeInfo) {
-      asset.durationInSeconds = (timeInfo.end - timeInfo.start) / asset.playbackRate;
-    }
-    asset.duration = asset.endInVideo - asset.startInVideo + 1;
-  });
-
-  return assets;
-}
-
-/**
- * Audio processor using Web Audio API
- */
-export class BrowserAudioProcessor {
-  private audioContext: AudioContext;
-
-  constructor(private sampleRate: number = 48000) {
-    this.audioContext = new AudioContext({ sampleRate });
-  }
-
-  /**
-   * Fetch and decode audio from a media source
-   */
-  async fetchAndDecodeAudio(src: string): Promise<AudioBuffer> {
-    const response = await fetch(src);
-    const arrayBuffer = await response.arrayBuffer();
-    return await this.audioContext.decodeAudioData(arrayBuffer);
-  }
-
-  /**
-   * Process audio asset with playback rate, volume, and timing
-   */
-  async processAudioAsset(
-    asset: MediaAsset,
-    fps: number,
-    totalFrames: number
-  ): Promise<AudioBuffer> {
-    const audioBuffer = await this.fetchAndDecodeAudio(asset.src);
-    
-    const duration = totalFrames / fps;
-    const outputLength = Math.ceil(duration * this.sampleRate);
-    const outputBuffer = this.audioContext.createBuffer(
-      2, // stereo
-      outputLength,
-      this.sampleRate
-    );
-
-    // Calculate timing
-    const startTime = asset.startInVideo / fps;
-    const trimLeft = asset.trimLeftInSeconds / asset.playbackRate;
-    const trimRight = trimLeft + asset.durationInSeconds;
-
-    // Process each channel
-    for (let channel = 0; channel < 2; channel++) {
-      const inputData = audioBuffer.getChannelData(Math.min(channel, audioBuffer.numberOfChannels - 1));
-      const outputData = outputBuffer.getChannelData(channel);
-
-      // Calculate sample positions
-      const startSample = Math.floor(startTime * this.sampleRate);
-      const trimLeftSample = Math.floor(trimLeft * this.sampleRate);
-      const trimRightSample = Math.floor(trimRight * this.sampleRate);
-
-      // Copy and process samples
-      for (let i = 0; i < outputData.length; i++) {
-        const outputTime = i / this.sampleRate;
-        const assetTime = outputTime - startTime;
-        
-        if (assetTime < 0 || assetTime >= asset.durationInSeconds) {
-          outputData[i] = 0; // Silence
-        } else {
-          // Apply playback rate
-          const inputSample = Math.floor((trimLeftSample + assetTime * asset.playbackRate * this.sampleRate));
-          if (inputSample >= 0 && inputSample < inputData.length) {
-            outputData[i] = inputData[inputSample] * asset.volume;
-          } else {
-            outputData[i] = 0;
-          }
-        }
-      }
-    }
-
-    return outputBuffer;
-  }
-
-  /**
-   * Mix multiple audio buffers
-   */
-  mixAudioBuffers(buffers: AudioBuffer[]): AudioBuffer {
-    if (buffers.length === 0) {
-      return this.audioContext.createBuffer(2, 1, this.sampleRate);
-    }
-
-    const maxLength = Math.max(...buffers.map(b => b.length));
-    const mixedBuffer = this.audioContext.createBuffer(2, maxLength, this.sampleRate);
-
-    for (let channel = 0; channel < 2; channel++) {
-      const mixedData = mixedBuffer.getChannelData(channel);
-      
-      buffers.forEach(buffer => {
-        const channelData = buffer.getChannelData(Math.min(channel, buffer.numberOfChannels - 1));
-        for (let i = 0; i < channelData.length; i++) {
-          mixedData[i] = (mixedData[i] || 0) + channelData[i] / buffers.length;
-        }
-      });
-    }
-
-    return mixedBuffer;
-  }
-
-  /**
-   * Convert AudioBuffer to WAV format
-   */
-  audioBufferToWav(buffer: AudioBuffer): ArrayBuffer {
-    const numberOfChannels = buffer.numberOfChannels;
-    const sampleRate = buffer.sampleRate;
-    const format = 1; // PCM
-    const bitDepth = 16;
-
-    const bytesPerSample = bitDepth / 8;
-    const blockAlign = numberOfChannels * bytesPerSample;
-
-    const data = new Float32Array(buffer.length * numberOfChannels);
-    for (let channel = 0; channel < numberOfChannels; channel++) {
-      const channelData = buffer.getChannelData(channel);
-      for (let i = 0; i < buffer.length; i++) {
-        data[i * numberOfChannels + channel] = channelData[i];
-      }
-    }
-
-    const dataLength = data.length * bytesPerSample;
-    const headerLength = 44;
-    const wav = new ArrayBuffer(headerLength + dataLength);
-    const view = new DataView(wav);
-
-    // Write WAV header
-    const writeString = (offset: number, string: string) => {
-      for (let i = 0; i < string.length; i++) {
-        view.setUint8(offset + i, string.charCodeAt(i));
-      }
-    };
-
-    writeString(0, 'RIFF');
-    view.setUint32(4, 36 + dataLength, true);
-    writeString(8, 'WAVE');
-    writeString(12, 'fmt ');
-    view.setUint32(16, 16, true); // fmt chunk size
-    view.setUint16(20, format, true);
-    view.setUint16(22, numberOfChannels, true);
-    view.setUint32(24, sampleRate, true);
-    view.setUint32(28, sampleRate * blockAlign, true);
-    view.setUint16(32, blockAlign, true);
-    view.setUint16(34, bitDepth, true);
-    writeString(36, 'data');
-    view.setUint32(40, dataLength, true);
-
-    // Write audio data
-    const volume = 0.8;
-    let offset = 44;
-    for (let i = 0; i < data.length; i++) {
-      const sample = Math.max(-1, Math.min(1, data[i]));
-      view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7FFF, true);
-      offset += 2;
-    }
-
-    return wav;
-  }
-
-  async close() {
-    await this.audioContext.close();
-  }
-}

package/src/audio/audio-video-muxer.ts

@@ -1,79 +0,0 @@
-/**
- * Browser-based audio/video muxing using FFmpeg.wasm (main thread)
- * Loads core files from local public/ffmpeg directory
- */
-
-export interface MuxerOptions {
-  videoBlob: Blob;
-  audioBuffer: ArrayBuffer;
-  fps: number;
-  width: number;
-  height: number;
-}
-
-/**
- * Mux audio and video using FFmpeg.wasm in main thread
- * Core files loaded from /ffmpeg/ (no CDN, no CORS issues)
- */
-export async function muxAudioVideo(options: MuxerOptions): Promise<Blob> {
-  try {
-    console.log('🎬 Starting FFmpeg.wasm muxing (main thread)...');
-    
-    // Import from installed packages (bundled by Vite)
-    const { FFmpeg } = await import('@ffmpeg/ffmpeg');
-    const { fetchFile, toBlobURL } = await import('@ffmpeg/util');
-    
-    const ffmpeg = new FFmpeg();
-    
-    ffmpeg.on('log', ({ message }) => {
-      console.log('[FFmpeg]', message);
-    });
-    
-    ffmpeg.on('progress', ({ progress }) => {
-      console.log(`[FFmpeg] Progress: ${(progress * 100).toFixed(1)}%`);
-    });
-    
-    console.log('[FFmpeg] Loading core from /ffmpeg/...');
-    
-    // Load from LOCAL files in public/ffmpeg (no CDN!)
-    // Note: FFmpeg 0.12.x has worker embedded in core.js, no separate workerURL needed
-    await ffmpeg.load({
-      coreURL: await toBlobURL('/ffmpeg/ffmpeg-core.js', 'text/javascript'),
-      wasmURL: await toBlobURL('/ffmpeg/ffmpeg-core.wasm', 'application/wasm'),
-    });
-    
-    console.log('✅ FFmpeg.wasm loaded');
-    
-    // Write input files
-    console.log('[FFmpeg] Writing input files...');
-    await ffmpeg.writeFile('video.mp4', await fetchFile(options.videoBlob));
-    await ffmpeg.writeFile('audio.wav', new Uint8Array(options.audioBuffer));
-    
-    console.log('[FFmpeg] Muxing audio and video...');
-    
-    // Mux video and audio
-    await ffmpeg.exec([
-      '-i', 'video.mp4',
-      '-i', 'audio.wav',
-      '-c:v', 'copy',
-      '-c:a', 'aac',
-      '-b:a', '192k',
-      '-shortest',
-      'output.mp4'
-    ]);
-    
-    console.log('[FFmpeg] Reading output file...');
-    
-    // Read output
-    const data = await ffmpeg.readFile('output.mp4');
-    const muxedBlob = new Blob([data], { type: 'video/mp4' });
-    
-    console.log(`✅ Muxed video with audio: ${(muxedBlob.size / 1024 / 1024).toFixed(2)} MB`);
-    
-    return muxedBlob;
-  } catch (error) {
-    console.error('❌ FFmpeg.wasm muxing failed:', error);
-    console.warn('⚠️ Returning video-only');
-    return options.videoBlob;
-  }
-}