@stinkycomputing/web-live-player 0.1.0

package/README.md

@@ -0,0 +1,323 @@
+# Web Live Player
+
+A framework-agnostic video streaming library with support for MoQ (Media over QUIC) and WebSocket backends.
+
+## Features
+
+- **WebCodecs-based decoding** - Hardware-accelerated video decoding
+- **MoQ support** - Native Media over QUIC protocol support via `stinky-moq-js`
+- **Pluggable stream sources** - Use dependency injection to provide video data from any transport
+- **Frame scheduling** - Automatic buffering and drift correction for smooth playback
+- **No framework dependencies** - Works with vanilla JS, React, Three.js, or any other framework
+
+## Installation
+
+```bash
+npm install @stinkycomputing/web-live-player
+```
+
+For MoQ support, also install:
+
+```bash
+npm install stinky-moq-js
+```
+
+## Quick Start
+
+### Using with MoQ (Standalone)
+
+```typescript
+import { createPlayer, createStandaloneMoQSource } from '@stinkycomputing/web-live-player';
+
+// Create player
+const player = createPlayer({
+  preferredDecoder: 'webcodecs-hw',
+  bufferSizeFrames: 3,
+});
+
+// Create MoQ source
+const moqSource = createStandaloneMoQSource({
+  relayUrl: 'https://moq-relay.example.com',
+  namespace: 'live/stream',
+  subscriptions: [
+    { trackName: 'video', streamType: 'video' },
+  ],
+});
+
+// Connect and play
+await moqSource.connect();
+player.setStreamSource(moqSource);
+player.setTrackFilter('video');
+player.play();
+
+// Render loop
+function render(timestamp) {
+  const frame = player.getVideoFrame(timestamp);
+  if (frame) {
+    ctx.drawImage(frame, 0, 0);
+  }
+  requestAnimationFrame(render);
+}
+requestAnimationFrame(render);
+```
+
+### Using with Elmo's MoQSession
+
+```typescript
+import { createPlayer } from '@stinkycomputing/web-live-player';
+import { MoQDiscoveryUtils } from '@elmo/core';
+
+// Find session from Elmo's node tree
+// MoQSessionNode implements IStreamSource directly
+const session = MoQDiscoveryUtils.findMoQSession(currentNode, 'my-session');
+
+// Create and configure player - session can be used directly as stream source
+const player = createPlayer();
+player.setStreamSource(session);
+player.setTrackFilter('video-track');
+player.play();
+```
+
+### Custom Stream Source
+
+```typescript
+import { createPlayer, IStreamSource, BaseStreamSource } from '@stinkycomputing/web-live-player';
+
+class MyCustomSource extends BaseStreamSource {
+  async connect() {
+    // Your connection logic
+    this._connected = true;
+    this.emit('connected');
+  }
+  
+  // Call this when you receive video data
+  handleVideoData(trackName: string, data: ParsedData) {
+    this.emit('data', {
+      trackName,
+      streamType: 'video',
+      data,
+    });
+  }
+}
+
+const source = new MyCustomSource();
+await source.connect();
+
+const player = createPlayer();
+player.setStreamSource(source);
+player.play();
+```
+
+## API Reference
+
+### `createPlayer(config?)`
+
+Creates a new player instance.
+
+**Config options:**
+- `preferredDecoder`: `'webcodecs-hw'` | `'webcodecs-sw'` | `'wasm'` - Decoder preference
+- `bufferSizeFrames`: `number` - Target buffer size (default: 3)
+- `debugLogging`: `boolean` - Enable debug logging
+
+### `LiveVideoPlayer`
+
+Main player class.
+
+**Methods:**
+- `setStreamSource(source: IStreamSource)` - Set the stream data source
+- `setTrackFilter(trackName: string)` - Filter for specific track
+- `play()` - Start playback
+- `pause()` - Pause playback
+- `getVideoFrame(timestampMs: number)` - Get frame for current render timestamp
+- `getStats()` - Get playback statistics
+- `dispose()` - Clean up resources
+
+**Events:**
+- `frame` - Emitted when a frame is decoded
+- `metadata` - Emitted when stream metadata is received
+- `statechange` - Emitted when player state changes
+- `error` - Emitted on errors
+
+### `IStreamSource`
+
+Interface for stream data sources.
+
+**Events to emit:**
+- `data` - Stream data event with `{ trackName, streamType, data }`
+- `connected` - When connected
+- `disconnected` - When disconnected
+- `error` - On errors
+
+## Rendering Frames to Canvas
+
+The player returns `VideoFrame` objects that can be rendered in multiple ways:
+
+### Basic Canvas Rendering
+
+```typescript
+const canvas = document.getElementById('video-canvas') as HTMLCanvasElement;
+const ctx = canvas.getContext('2d')!;
+
+function render(timestamp: number) {
+  const frame = player.getVideoFrame(timestamp);
+  if (frame) {
+    // Resize canvas to match video dimensions
+    if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+      canvas.width = frame.displayWidth;
+      canvas.height = frame.displayHeight;
+    }
+    
+    // Draw the frame
+    ctx.drawImage(frame, 0, 0);
+    
+    // IMPORTANT: Close the frame when done to release memory
+    frame.close();
+  }
+  requestAnimationFrame(render);
+}
+requestAnimationFrame(render);
+```
+
+### WebGL / Three.js Rendering
+
+For GPU-accelerated rendering (e.g., in Three.js):
+
+```typescript
+// Create a texture
+const texture = new THREE.Texture();
+texture.minFilter = THREE.LinearFilter;
+texture.magFilter = THREE.LinearFilter;
+texture.colorSpace = THREE.SRGBColorSpace;
+
+// In your render loop
+function render(timestamp: number) {
+  const frame = player.getVideoFrame(timestamp);
+  if (frame) {
+    // Update texture with the VideoFrame
+    texture.image = frame;
+    texture.needsUpdate = true;
+    
+    // Close previous frame if stored
+    if (lastFrame) lastFrame.close();
+    lastFrame = frame;
+  }
+  
+  renderer.render(scene, camera);
+  requestAnimationFrame(render);
+}
+```
+
+### Handling YUV Frames (WASM Decoder)
+
+When using the WASM decoder, the library automatically converts YUV frames to `VideoFrame` objects using the browser's native I420 support. The GPU handles YUV→RGB conversion, so you can use the same rendering code regardless of decoder:
+
+```typescript
+// The player always returns VideoFrame, even with WASM decoder
+const frame = player.getVideoFrame(timestamp);
+if (frame) {
+  ctx.drawImage(frame, 0, 0);
+  frame.close();
+}
+```
+
+If you need raw YUV data for custom processing, you can access the `WasmDecoder` directly:
+
+```typescript
+import { WasmDecoder } from '@stinkycomputing/web-live-player';
+
+const decoder = new WasmDecoder({
+  onFrameDecoded: (yuvFrame) => {
+    // yuvFrame has: { y, u, v, width, height, stride, chromaStride, chromaHeight, timestamp }
+    // Process raw YUV data here
+  },
+});
+```
+
+### Best Practices
+
+1. **Always close VideoFrames** - Call `frame.close()` when done to prevent memory leaks
+2. **Check for null frames** - `getVideoFrame()` returns null when no frame is ready
+3. **Use performance.now()** - Pass accurate timestamps for proper frame scheduling
+4. **Handle resize** - Update canvas dimensions when video dimensions change
+
+
+## Bundler Configuration
+
+### WASM Decoder (tinyh264)
+
+The WASM decoder uses `tinyh264` which requires special bundler configuration for its Web Worker and WASM assets.
+
+#### Vite
+
+Add the following to your `vite.config.ts`:
+
+```typescript
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+  // Handle tinyh264's .asset files as URLs
+  assetsInclude: ['**/*.asset'],
+  
+  // Ensure worker files are bundled correctly
+  worker: {
+    format: 'es',
+  },
+});
+```
+
+#### Webpack
+
+For Webpack, you may need to configure asset handling:
+
+```javascript
+module.exports = {
+  module: {
+    rules: [
+      {
+        test: /\.asset$/,
+        type: 'asset/resource',
+      },
+    ],
+  },
+};
+```
+
+### WebCodecs Decoder (Recommended)
+
+If you only need WebCodecs-based decoding (hardware or software), no special bundler configuration is required. Simply use:
+
+```typescript
+const player = createPlayer({
+  preferredDecoder: 'webcodecs-hw', // or 'webcodecs-sw'
+});
+```
+
+## Demo
+
+Run the demo application:
+
+```bash
+cd video-player
+npm install
+npm run dev
+```
+
+Open http://localhost:3001 to see the demo.
+
+## Building
+
+Build the library:
+
+```bash
+npm run build
+```
+
+Build the demo:
+
+```bash
+npm run build:demo
+```
+
+## License
+
+MIT

package/dist/audio/file-audio-player.d.ts

@@ -0,0 +1,60 @@
+/**
+ * File Audio Player
+ *
+ * Decodes and plays audio from MP4 files using WebCodecs AudioDecoder
+ * and AudioWorklet for real-time playback.
+ */
+export interface AudioPlayerConfig {
+    sampleRate?: number;
+    channels?: number;
+}
+export declare class FileAudioPlayer {
+    private ctx;
+    private decoder;
+    private workletNode;
+    private gainNode;
+    private initialized;
+    private targetSampleRate;
+    private codecConfig;
+    constructor(context: AudioContext, config?: AudioPlayerConfig);
+    /**
+     * Initialize the audio player with codec info from MP4
+     */
+    init(codec: string, sampleRate: number, numberOfChannels: number, description?: Uint8Array): Promise<void>;
+    /**
+     * Handle decoded audio frame
+     */
+    private handleDecodedFrame;
+    /**
+     * Resample audio to target sample rate
+     */
+    private resampleAudio;
+    /**
+     * Decode an audio sample
+     */
+    decode(data: Uint8Array, timestamp: number, duration: number): void;
+    /**
+     * Start audio playback
+     */
+    start(): void;
+    /**
+     * Stop audio playback
+     */
+    stop(): void;
+    /**
+     * Clear audio buffers
+     */
+    clear(): void;
+    /**
+     * Reset timing for seamless loop (clears worklet buffer)
+     */
+    resetTiming(): void;
+    /**
+     * Set volume (0-1)
+     */
+    setVolume(volume: number): void;
+    /**
+     * Dispose of resources
+     */
+    dispose(): void;
+}

package/dist/audio/live-audio-player.d.ts

@@ -0,0 +1,64 @@
+import { HeaderCodecData } from '../protocol/sesame-binary-protocol';
+
+export interface LiveAudioConfig {
+    bufferDelayMs?: number;
+}
+export declare class LiveAudioPlayer {
+    private ctx;
+    private decoder;
+    private workletNode;
+    private gainNode;
+    private initialized;
+    private targetSampleRate;
+    private startTime;
+    private bufferDelayMs;
+    constructor(context: AudioContext, config?: LiveAudioConfig);
+    /**
+     * Initialize with codec info from stream header
+     */
+    init(codecData?: HeaderCodecData): Promise<void>;
+    /**
+     * Build decoder config based on codec type
+     */
+    private buildDecoderConfig;
+    /**
+     * Handle decoded audio frame
+     */
+    private handleDecodedFrame;
+    /**
+     * Resample audio to target sample rate
+     */
+    private resampleAudio;
+    /**
+     * Queue encoded audio data for decoding
+     */
+    decode(data: Uint8Array, pts?: bigint): void;
+    /**
+     * Set buffer delay for A/V sync
+     */
+    setBufferDelay(delayMs: number): void;
+    /**
+     * Start audio playback
+     */
+    start(): void;
+    /**
+     * Stop audio playback
+     */
+    stop(): void;
+    /**
+     * Clear audio buffers (for seek/reset)
+     */
+    clear(): void;
+    /**
+     * Reset timing reference
+     */
+    resetTiming(): void;
+    /**
+     * Set volume (0-1)
+     */
+    setVolume(volume: number): void;
+    /**
+     * Dispose resources
+     */
+    dispose(): void;
+}

package/dist/decoders/decoder-interface.d.ts

@@ -0,0 +1,63 @@
+import { ParsedData, HeaderCodecData } from '../protocol/sesame-binary-protocol';
+import { YUVFrame } from '../types';
+
+/**
+ * Decoder state (matches WebCodecs VideoDecoder states)
+ */
+export type DecoderState = 'unconfigured' | 'configuring' | 'configured' | 'closed';
+/**
+ * Frame output from a decoder - can be VideoFrame or YUVFrame
+ */
+export type DecodedFrame = VideoFrame | YUVFrame;
+/**
+ * Configuration options for decoder initialization
+ */
+export interface VideoDecoderOptions {
+    /** Callback for decoded frames */
+    onFrameDecoded?: (frame: DecodedFrame) => void;
+    /** Callback for errors */
+    onError?: (error: Error) => void;
+    /** Callback when decoder queue overflows */
+    onQueueOverflow?: (queueSize: number) => void;
+    /** Maximum decoder queue size before overflow callback */
+    maxQueueSize?: number;
+}
+/**
+ * Common interface for video decoders
+ */
+export interface IVideoDecoder {
+    /**
+     * Current decoder state
+     */
+    readonly state: DecoderState | string;
+    /**
+     * Configure the decoder for a specific codec
+     * @param codecData - Codec configuration from stream header
+     */
+    configure(codecData: HeaderCodecData): Promise<void>;
+    /**
+     * Decode a binary packet from the stream
+     * @param data - Parsed stream data containing header and payload
+     */
+    decodeBinary(data: ParsedData): void;
+    /**
+     * Flush pending frames (synchronous reset)
+     */
+    flush(): void;
+    /**
+     * Reset the decoder to configured state (ready for new keyframe)
+     */
+    reset(): void;
+    /**
+     * Dispose the decoder and release resources
+     */
+    dispose(): void;
+}
+/**
+ * Type guard to check if a frame is a VideoFrame
+ */
+export declare function isVideoFrame(frame: DecodedFrame): frame is VideoFrame;
+/**
+ * Type guard to check if a frame is a YUVFrame
+ */
+export declare function isYUVFrame(frame: DecodedFrame): frame is YUVFrame;

package/dist/decoders/wasm-decoder.d.ts

@@ -0,0 +1,68 @@
+import { ParsedData } from '../protocol/sesame-binary-protocol';
+import { YUVFrame } from '../types';
+import { IVideoDecoder } from './decoder-interface';
+
+export type { YUVFrame } from '../types';
+export interface WasmDecoderConfig {
+    onFrameDecoded?: (frame: YUVFrame) => void;
+    onError?: (error: Error) => void;
+    onQueueOverflow?: (queueSize: number) => void;
+    maxQueueSize?: number;
+}
+export declare class WasmDecoder implements IVideoDecoder {
+    private worker?;
+    private _queueSize;
+    private pendingTimestamps;
+    private pendingFrames;
+    private maxQueueSize;
+    private onFrameDecoded?;
+    private onError?;
+    private onQueueOverflow?;
+    configured: boolean;
+    constructor(config?: WasmDecoderConfig);
+    get queueSize(): number;
+    /**
+     * Get decoder state (for compatibility with WebCodecsDecoder)
+     */
+    get state(): string;
+    /**
+     * Configure the decoder (initializes the worker)
+     */
+    configure(_codecData: {
+        codec_type: number;
+        width: number;
+        height: number;
+    }): Promise<void>;
+    /**
+     * Decode a binary packet (same interface as WebCodecsDecoder)
+     */
+    decodeBinary(data: ParsedData): void;
+    /**
+     * Process next frame from queue
+     */
+    private decodeNext;
+    /**
+     * Send frame to worker for decoding
+     */
+    private decode;
+    /**
+     * Handle decoded picture from worker
+     */
+    private handlePictureReady;
+    /**
+     * Flush the decoder (clear pending frames)
+     */
+    flush(): void;
+    /**
+     * Synchronous flush
+     */
+    flushSync(): void;
+    /**
+     * Reset the decoder (same as flush for WASM decoder)
+     */
+    reset(): void;
+    /**
+     * Dispose of the decoder
+     */
+    dispose(): void;
+}

package/dist/decoders/wasm-worker/H264NALDecoder.worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/decoders/webcodecs-decoder.d.ts

@@ -0,0 +1,82 @@
+import { ParsedData, HeaderCodecData } from '../protocol/sesame-binary-protocol';
+import { Logger } from '../types';
+import { IVideoDecoder } from './decoder-interface';
+
+export interface DecoderConfig {
+    preferHardware?: boolean;
+    logger?: Logger;
+    onFrameDecoded?: (frame: VideoFrame) => void;
+    /** Alias for onFrameDecoded */
+    onFrame?: (frame: VideoFrame) => void;
+    onError?: (error: Error) => void;
+    onQueueOverflow?: (queueSize: number) => void;
+    maxQueueSize?: number;
+}
+/**
+ * Sample data for decoding (from file demuxer)
+ */
+export interface SampleData {
+    data: Uint8Array;
+    timestamp: number;
+    duration?: number;
+    isKeyframe: boolean;
+}
+/**
+ * WebCodecs-based video decoder
+ */
+export declare class WebCodecsDecoder implements IVideoDecoder {
+    private decoder;
+    private config;
+    private flushing;
+    private logger;
+    private maxQueueSize;
+    onFrameDecoded?: (frame: VideoFrame) => void;
+    onError?: (error: Error) => void;
+    onQueueOverflow?: (queueSize: number) => void;
+    chunksSentToDecoder: number;
+    framesDecoded: number;
+    constructor(options?: DecoderConfig);
+    get decodeQueueSize(): number;
+    get state(): string;
+    private createDecoder;
+    /**
+     * Configure the decoder for a specific codec (from HeaderCodecData)
+     */
+    configure(codecData: HeaderCodecData, preferHardware?: boolean): Promise<void>;
+    /**
+     * Configure the decoder with VideoDecoderConfig directly
+     */
+    configure(config: VideoDecoderConfig): Promise<void>;
+    /**
+     * Configure with VideoDecoderConfig
+     */
+    private configureWithConfig;
+    /**
+     * Configure with HeaderCodecData (from binary protocol)
+     */
+    private configureWithCodecData;
+    /**
+     * Decode a binary packet
+     */
+    decodeBinary(data: ParsedData): void;
+    /**
+     * Decode a sample (from file demuxer)
+     */
+    decode(sample: SampleData): void;
+    /**
+     * Flush the decoder (async - waits for pending frames)
+     */
+    flush(): Promise<void>;
+    /**
+     * Synchronous flush - resets decoder immediately (used for overflow recovery)
+     */
+    flushSync(): void;
+    /**
+     * Reset the decoder
+     */
+    reset(): void;
+    /**
+     * Dispose the decoder
+     */
+    dispose(): void;
+}

package/dist/index.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Web Live Player - A framework-agnostic video streaming library
+ *
+ * Supports multiple stream sources through dependency injection:
+ * - MoQSession (from Elmo or standalone)
+ * - WebSocket connections
+ * - Custom stream sources
+ * - MP4 file playback (client-side demuxing with mp4box)
+ */
+export * from './types';
+export * from './sources/stream-source';
+export { BasePlayer } from './player/base-player';
+export type { BasePlayerConfig } from './player/base-player';
+export { LiveVideoPlayer, createPlayer } from './player/live-player';
+export type { PlayerConfig, PlayerStats, PlayerState } from './player/live-player';
+export { FileVideoPlayer, createFilePlayer } from './player/file-player';
+export type { FilePlayerConfig, FilePlayerState, FilePlayerStats, FilePlayMode } from './player/file-player';
+export { createStandaloneMoQSource, StandaloneMoQSource } from './sources/standalone-moq-source';
+export { WebSocketSource, createWebSocketSource } from './sources/websocket-source';
+export type { WebSocketSourceConfig, VideoMetadata } from './sources/websocket-source';
+export { MP4FileSource } from './sources/mp4-file-source';
+export type { MP4FileInfo, DecodableSample, MP4FileSourceEvents } from './sources/mp4-file-source';
+export { WebCodecsDecoder } from './decoders/webcodecs-decoder';
+export type { SampleData } from './decoders/webcodecs-decoder';
+export { WasmDecoder } from './decoders/wasm-decoder';
+export type { YUVFrame, WasmDecoderConfig } from './decoders/wasm-decoder';
+export type { IVideoDecoder, DecoderState, DecodedFrame, VideoDecoderOptions } from './decoders/decoder-interface';
+export { isVideoFrame, isYUVFrame } from './decoders/decoder-interface';
+export { FileAudioPlayer } from './audio/file-audio-player';
+export { LiveAudioPlayer } from './audio/live-audio-player';
+export type { LiveAudioConfig } from './audio/live-audio-player';
+export { FrameScheduler } from './scheduling/frame-scheduler';
+export type { FrameTiming, LatencyStats, SchedulerStatus, SchedulerConfig } from './scheduling/frame-scheduler';
+export { SesameBinaryProtocol } from './protocol/sesame-binary-protocol';
+export type { ParsedData, HeaderData, HeaderCodecData } from './protocol/sesame-binary-protocol';