@twick/browser-render 0.15.6

package/AUDIO_IMPLEMENTATION.md

@@ -0,0 +1,217 @@
+# Audio Implementation Guide
+
+## Overview
+
+Browser-based audio processing for Twick video rendering, matching the server's FFmpeg implementation.
+
+## Architecture
+
+### 1. **Audio Processor** (`src/audio/audio-processor.ts`)
+- Mirrors server's `generate-audio.ts` logic
+- Uses Web Audio API instead of FFmpeg
+- Handles:
+  - Asset tracking across frames
+  - Audio extraction from video/audio elements
+  - Playback rate adjustment
+  - Volume control
+  - Audio trimming
+  - Multi-track mixing
+
+### 2. **Service Worker** (`public/audio-worker.js`)
+- Caches media assets for offline processing
+- Handles background audio extraction
+- Reduces network requests during rendering
+
+### 3. **Audio/Video Muxer** (`src/audio/audio-video-muxer.ts`)
+- Combines video and audio tracks
+- Two approaches:
+  - **mp4box.js**: Lightweight, browser-native
+  - **FFmpeg.wasm**: More robust, ~30MB bundle
+
+## Implementation Steps
+
+### Step 1: Install Dependencies
+
+```bash
+# For mp4box.js approach (recommended)
+npm install mp4box
+
+# OR for FFmpeg.wasm approach (more features, larger)
+npm install @ffmpeg/ffmpeg @ffmpeg/util
+```
+
+### Step 2: Register Service Worker
+
+```typescript
+// In your app's initialization
+if ('serviceWorker' in navigator) {
+  navigator.serviceWorker.register('/audio-worker.js')
+    .then(reg => console.log('Audio worker registered'))
+    .catch(err => console.error('Audio worker failed:', err));
+}
+```
+
+### Step 3: Enable Audio in Browser Renderer
+
+Update `browser-renderer.ts`:
+
+```typescript
+import { BrowserAudioProcessor, getAssetPlacement } from './audio/audio-processor';
+import { muxAudioVideo } from './audio/audio-video-muxer';
+
+// In BrowserWasmExporter.generateAudio():
+public async generateAudio(
+  assets: any[][],
+  startFrame: number,
+  endFrame: number,
+): Promise<ArrayBuffer | null> {
+  const processor = new BrowserAudioProcessor();
+  const assetPlacements = getAssetPlacement(assets);
+  
+  const processedBuffers: AudioBuffer[] = [];
+  for (const asset of assetPlacements) {
+    if (asset.volume > 0 && asset.playbackRate > 0) {
+      const buffer = await processor.processAudioAsset(
+        asset,
+        this.settings.fps || 30,
+        endFrame - startFrame
+      );
+      processedBuffers.push(buffer);
+    }
+  }
+  
+  const mixedBuffer = processor.mixAudioBuffers(processedBuffers);
+  const wavData = processor.audioBufferToWav(mixedBuffer);
+  
+  await processor.close();
+  return wavData;
+}
+```
+
+### Step 4: Collect Audio Assets During Rendering
+
+In `renderTwickVideoInBrowser()`:
+
+```typescript
+// Track media assets for each frame
+const mediaAssets: AssetInfo[][] = [];
+
+for (let frame = 0; frame < totalFrames; frame++) {
+  // ... existing rendering code ...
+  
+  // Collect media assets from current scene
+  const currentAssets = (renderer as any).playback.currentScene.getMediaAssets?.() || [];
+  mediaAssets.push(currentAssets);
+}
+
+// Generate audio after video rendering
+const audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
+
+// Mux audio and video
+if (audioData) {
+  const finalBlob = await muxAudioVideo({
+    videoBlob,
+    audioBuffer: audioData,
+    fps,
+    width,
+    height
+  });
+  return finalBlob;
+}
+```
+
+## API Parity with Server
+
+| Feature | Server (FFmpeg) | Browser (Web Audio) | Status |
+|---------|-----------------|---------------------|--------|
+| Asset Tracking | `getAssetPlacement()` | `getAssetPlacement()` | ✅ Ready |
+| Audio Extraction | FFmpeg decode | `decodeAudioData()` | ✅ Ready |
+| Playback Rate | `atempo` filter | Sample interpolation | ✅ Ready |
+| Volume | `volume` filter | Gain multiplication | ✅ Ready |
+| Trimming | `atrim` filter | Sample slicing | ✅ Ready |
+| Mixing | `amix` filter | Buffer mixing | ✅ Ready |
+| WAV Encoding | FFmpeg encode | Manual WAV encoding | ✅ Ready |
+| Muxing | FFmpeg merge | mp4box.js / FFmpeg.wasm | ⚠️ Needs library |
+
+## Performance Considerations
+
+### Memory Usage
+- Web Audio API decodes entire audio files into memory
+- Large video files can cause memory issues
+- Consider chunked processing for long videos
+
+### Processing Time
+- Audio processing adds 20-50% to render time
+- Service worker caching helps with repeated renders
+- Consider showing separate progress for video/audio
+
+### Browser Limits
+- Chrome: ~2GB audio buffer limit
+- Safari: Stricter memory limits
+- Firefox: Better memory management but slower
+
+## Example Usage
+
+```typescript
+import { useBrowserRenderer } from '@twick/browser-render';
+
+function VideoRenderer() {
+  const { render, progress } = useBrowserRenderer({
+    width: 1920,
+    height: 1080,
+    fps: 30,
+    includeAudio: true, // Enable audio processing
+  });
+
+  const handleRender = async () => {
+    const videoBlob = await render({
+      input: {
+        properties: { width: 1920, height: 1080, fps: 30 },
+        tracks: [
+          {
+            type: 'element',
+            elements: [
+              {
+                type: 'video',
+                src: 'https://example.com/video.mp4',
+                // Audio will be automatically extracted and included
+              }
+            ]
+          }
+        ]
+      }
+    });
+  };
+
+  return <button onClick={handleRender}>Render with Audio</button>;
+}
+```
+
+## Troubleshooting
+
+### No Audio in Output
+1. Check if `includeAudio: true` is set
+2. Verify service worker is registered
+3. Check browser console for Web Audio API errors
+4. Ensure video sources have audio tracks
+
+### Memory Errors
+1. Reduce video quality/resolution
+2. Process shorter segments
+3. Clear service worker cache
+4. Use FFmpeg.wasm with streaming
+
+### Performance Issues
+1. Use service worker caching
+2. Reduce number of audio tracks
+3. Lower audio sample rate (default: 48kHz)
+4. Consider server-side rendering for production
+
+## Future Enhancements
+
+- [ ] Streaming audio processing (chunks)
+- [ ] Web Workers for parallel processing
+- [ ] Real-time audio preview
+- [ ] Audio effects (reverb, EQ, etc.)
+- [ ] WASM-based audio processing for better performance
+- [ ] Support for more audio formats

package/README.md

@@ -0,0 +1,151 @@
+# @twick/browser-render
+
+Browser-native video rendering for Twick using WebCodecs API.
+
+## Installation
+
+```bash
+npm install @twick/browser-render
+```
+
+## Usage
+
+### Basic Example
+
+```typescript
+import { renderTwickVideoInBrowser } from '@twick/browser-render';
+
+const videoBlob = await renderTwickVideoInBrowser({
+  variables: {
+    input: {
+      properties: { width: 1920, height: 1080, fps: 30 },
+      tracks: [
+        // Your tracks configuration
+      ]
+    }
+  },
+  settings: {
+    width: 1920,
+    height: 1080,
+    fps: 30,
+    onProgress: (progress) => console.log(`${(progress * 100).toFixed(0)}%`)
+  }
+});
+
+// Download the video
+const url = URL.createObjectURL(videoBlob);
+const a = document.createElement('a');
+a.href = url;
+a.download = 'video.mp4';
+a.click();
+```
+
+### React Hook
+
+```tsx
+import { useBrowserRenderer } from '@twick/browser-render';
+
+function VideoRenderer() {
+  const { render, progress, isRendering, videoBlob, download } = useBrowserRenderer({
+    width: 1920,
+    height: 1080,
+    fps: 30,
+    autoDownload: false
+  });
+
+  const handleRender = async () => {
+    await render({
+      input: {
+        properties: { width: 1920, height: 1080, fps: 30 },
+        tracks: [/* ... */]
+      }
+    });
+  };
+
+  return (
+    <div>
+      <button onClick={handleRender} disabled={isRendering}>
+        Render Video
+      </button>
+      {isRendering && <progress value={progress} max={1} />}
+      {videoBlob && <button onClick={download}>Download</button>}
+    </div>
+  );
+}
+```
+
+## Configuration
+
+### BrowserRenderConfig
+
+```typescript
+{
+  projectFile?: Project;     // Optional custom project (defaults to @twick/visualizer)
+  variables: {
+    input: any;              // Required: project input data
+    playerId?: string;
+    [key: string]: any;
+  };
+  settings?: {
+    width?: number;          // Video width (default: 1920)
+    height?: number;         // Video height (default: 1080)
+    fps?: number;            // Frames per second (default: 30)
+    quality?: 'low' | 'medium' | 'high';
+    range?: [number, number]; // [start, end] in seconds
+    onProgress?: (progress: number) => void;
+    onComplete?: (videoBlob: Blob) => void;
+    onError?: (error: Error) => void;
+  };
+}
+```
+
+## Custom Project
+
+```typescript
+import myProject from './my-project';
+
+const videoBlob = await renderTwickVideoInBrowser({
+  projectFile: myProject,  // Must be an imported Project object
+  variables: { input: {...} }
+});
+```
+
+**Note**: String paths only work in Node.js. In the browser, you must import and pass the Project object directly.
+
+## WASM Setup
+
+Copy the WASM file to your public directory:
+
+```bash
+cp node_modules/mp4-wasm/dist/mp4-wasm.wasm public/
+```
+
+Or configure Vite to serve it:
+
+```typescript
+// vite.config.ts
+export default defineConfig({
+  assetsInclude: ['**/*.wasm']
+});
+```
+
+## Limitations
+
+- **Audio**: Audio processing is not yet implemented. Only video encoding is supported.
+- **Browser Support**: Requires WebCodecs API (Chrome 94+, Edge 94+)
+
+## API Comparison
+
+This package follows the same API as the server renderer (`@twick/renderer`):
+
+| Feature | Server | Browser |
+|---------|--------|---------|
+| Duration Calculation | `renderer.getNumberOfFrames()` | ✅ Same |
+| Playback State | `PlaybackState.Rendering` | ✅ Same |
+| Frame Progression | `playback.progress()` | ✅ Same |
+| Variables Assignment | `project.variables = {...}` | ✅ Same |
+| Audio Support | ✅ FFmpeg | ❌ Not yet |
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@twick/browser-render",
+  "version": "0.15.6",
+  "license": "https://github.com/ncounterspecialist/twick/blob/main/LICENSE.md",
+  "description": "Browser-native video rendering for Twick using WebCodecs API",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test:browser": "tsx src/test-browser-render.ts",
+    "clean": "rimraf dist"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@twick/core": "^0.15.6",
+    "@twick/visualizer": "0.15.6",
+    "mp4-wasm": "^1.0.6",
+    "mp4box": "^0.5.2",
+    "@ffmpeg/ffmpeg": "^0.12.10",
+    "@ffmpeg/util": "^0.12.1",
+    "@ffmpeg/core": "^0.12.6"
+  },
+  "peerDependencies": {
+    "react": ">=17.0.0"
+  },
+  "peerDependenciesMeta": {
+    "react": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.0",
+    "@types/react": "^18.2.0",
+    "rimraf": "^5.0.5",
+    "tsup": "^8.0.0",
+    "tsx": "^4.7.0",
+    "typescript": "5.4.2"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/package.json.bak

@@ -0,0 +1,53 @@
+{
+  "name": "@twick/browser-render",
+  "version": "0.15.6",
+  "license": "https://github.com/ncounterspecialist/twick/blob/main/LICENSE.md",
+  "description": "Browser-native video rendering for Twick using WebCodecs API",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test:browser": "tsx src/test-browser-render.ts",
+    "clean": "rimraf dist"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@twick/core": "^0.15.6",
+    "@twick/visualizer": "0.15.6",
+    "mp4-wasm": "^1.0.6",
+    "mp4box": "^0.5.2",
+    "@ffmpeg/ffmpeg": "^0.12.10",
+    "@ffmpeg/util": "^0.12.1",
+    "@ffmpeg/core": "^0.12.6"
+  },
+  "peerDependencies": {
+    "react": ">=17.0.0"
+  },
+  "peerDependenciesMeta": {
+    "react": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.0",
+    "@types/react": "^18.2.0",
+    "rimraf": "^5.0.5",
+    "tsup": "^8.0.0",
+    "tsx": "^4.7.0",
+    "typescript": "5.4.2"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/public/audio-worker.js

@@ -0,0 +1,95 @@
+/**
+ * Service Worker for audio processing
+ * Handles audio extraction and processing in the background
+ */
+
+const CACHE_NAME = 'twick-audio-cache-v1';
+const AUDIO_CACHE = 'audio-assets';
+
+self.addEventListener('install', (event) => {
+  console.log('Audio Service Worker installed');
+  self.skipWaiting();
+});
+
+self.addEventListener('activate', (event) => {
+  console.log('Audio Service Worker activated');
+  event.waitUntil(self.clients.claim());
+});
+
+/**
+ * Intercept audio/video requests for processing
+ */
+self.addEventListener('fetch', (event) => {
+  const url = new URL(event.request.url);
+  
+  // Check if this is a media request we should cache
+  const isMedia = /\.(mp4|webm|mp3|wav|ogg|m4a)$/i.test(url.pathname);
+  
+  if (isMedia) {
+    event.respondWith(
+      caches.open(AUDIO_CACHE).then((cache) => {
+        return cache.match(event.request).then((response) => {
+          if (response) {
+            return response;
+          }
+          
+          return fetch(event.request).then((networkResponse) => {
+            // Cache for future use
+            cache.put(event.request, networkResponse.clone());
+            return networkResponse;
+          });
+        });
+      })
+    );
+  }
+});
+
+/**
+ * Handle messages from main thread
+ */
+self.addEventListener('message', async (event) => {
+  const { type, data } = event.data;
+  
+  switch (type) {
+    case 'EXTRACT_AUDIO':
+      // Extract audio from video URL
+      const audioData = await extractAudio(data.url);
+      event.ports[0].postMessage({ type: 'AUDIO_EXTRACTED', data: audioData });
+      break;
+      
+    case 'CLEAR_CACHE':
+      await caches.delete(AUDIO_CACHE);
+      event.ports[0].postMessage({ type: 'CACHE_CLEARED' });
+      break;
+  }
+});
+
+/**
+ * Extract audio from video URL
+ */
+async function extractAudio(url) {
+  try {
+    const response = await fetch(url);
+    const blob = await response.blob();
+    
+    // Return blob for processing in main thread
+    // (Web Audio API is not available in service workers)
+    return {
+      url,
+      blob: await blobToArrayBuffer(blob),
+      size: blob.size
+    };
+  } catch (error) {
+    console.error('Failed to extract audio:', error);
+    throw error;
+  }
+}
+
+async function blobToArrayBuffer(blob) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(reader.result);
+    reader.onerror = reject;
+    reader.readAsArrayBuffer(blob);
+  });
+}