@unboundcx/video-sdk-client 1.1.0

package/VideoProcessor.js

@@ -0,0 +1,375 @@
+/**
+ * VideoProcessor - Handles video processing (blur, virtual backgrounds)
+ * Uses MediaPipe SelfieSegmentation for background removal
+ */
+
+import SelfieSegmentationPkg from '@mediapipe/selfie_segmentation';
+import { Logger } from './utils/Logger.js';
+
+// Handle CommonJS module in Vite
+const { SelfieSegmentation } = SelfieSegmentationPkg;
+
+export class VideoProcessor {
+	constructor(options = {}) {
+		this.logger = new Logger('VideoProcessor', options.debug);
+		this.selfieSegmentation = null;
+		this.isInitialized = false;
+		this.processing = false;
+		this.offScreenVideo = null;
+		this.offScreenCanvas = null;
+		this.canvasCtx = null;
+		this.backgroundImage = null;
+		this.animationFrameId = null;
+
+		// Edge refinement settings
+		this.edgeBlurAmount = 5; // Increased from 4px for smoother edges
+		this.lightWrapIntensity = 0.15; // How much background light spills onto edges
+	}
+
+	/**
+	 * Initialize MediaPipe SelfieSegmentation model
+	 */
+	async initialize() {
+		if (this.isInitialized) {
+			this.logger.info('VideoProcessor already initialized');
+			return;
+		}
+
+		try {
+			this.logger.info('Initializing MediaPipe SelfieSegmentation');
+
+			this.selfieSegmentation = new SelfieSegmentation({
+				locateFile: (file) => {
+					return `/mediapipe/selfie_segmentation/${file}`;
+				},
+			});
+
+			this.selfieSegmentation.setOptions({
+				modelSelection: 1, // 0 = general, 1 = landscape (better quality)
+				selfieMode: false, // Set to false for better non-mirrored results
+			});
+
+			await this.selfieSegmentation.initialize();
+
+			this.isInitialized = true;
+			this.logger.info('VideoProcessor initialized successfully');
+		} catch (error) {
+			this.logger.error('Failed to initialize VideoProcessor:', error);
+			throw error;
+		}
+	}
+
+	/**
+	 * Apply background effect to video stream
+	 * @param {MediaStream} videoStream - Input video stream
+	 * @param {Object} options - Background options
+	 * @param {string} options.type - 'blur' | 'image' | 'none'
+	 * @param {number} options.blurLevel - Blur amount in pixels (default: 8)
+	 * @param {string} options.imageUrl - Background image URL (for type: 'image')
+	 * @returns {Promise<MediaStream>} Processed video stream
+	 */
+	async applyBackground(videoStream, options = {}) {
+		const { type = 'none', blurLevel = 8, imageUrl = null } = options;
+
+		if (type === 'none') {
+			this.logger.info('No background effect - returning original stream');
+			return videoStream;
+		}
+
+		if (!this.isInitialized) {
+			await this.initialize();
+		}
+
+		this.logger.info('Applying background effect:', { type, blurLevel, imageUrl });
+
+		// Load background image if needed
+		if (type === 'image' && imageUrl) {
+			this.backgroundImage = await this._loadImage(imageUrl);
+		}
+
+		// Create off-screen video element
+		this.offScreenVideo = document.createElement('video');
+		this.offScreenVideo.srcObject = videoStream;
+		this.offScreenVideo.autoplay = true;
+		this.offScreenVideo.muted = true;
+		this.offScreenVideo.playsInline = true;
+
+		// Create off-screen canvas
+		this.offScreenCanvas = document.createElement('canvas');
+		this.canvasCtx = this.offScreenCanvas.getContext('2d', {
+			willReadFrequently: true,
+		});
+
+		// Wait for video metadata
+		await new Promise((resolve) => {
+			this.offScreenVideo.onloadedmetadata = () => {
+				this.offScreenCanvas.width = this.offScreenVideo.videoWidth;
+				this.offScreenCanvas.height = this.offScreenVideo.videoHeight;
+				this.logger.info('Video dimensions:', {
+					width: this.offScreenVideo.videoWidth,
+					height: this.offScreenVideo.videoHeight,
+				});
+				resolve();
+			};
+		});
+
+		// Wait for video to actually start playing
+		await this.offScreenVideo.play();
+
+		// Wait for first frame to be processed before returning stream
+		this.processing = true;
+		const firstFrameProcessed = await this._startProcessingAndWaitForFirstFrame(
+			type,
+			blurLevel,
+		);
+
+		if (!firstFrameProcessed) {
+			this.logger.warn('First frame not processed, returning anyway');
+		}
+
+		// Return canvas stream
+		const processedStream = this.offScreenCanvas.captureStream(30);
+		this.logger.info('Background effect applied successfully');
+
+		return processedStream;
+	}
+
+	/**
+	 * Stop processing and cleanup resources
+	 */
+	async cleanup() {
+		this.logger.info('Cleaning up VideoProcessor');
+
+		this.processing = false;
+
+		if (this.animationFrameId) {
+			cancelAnimationFrame(this.animationFrameId);
+			this.animationFrameId = null;
+		}
+
+		if (this.offScreenVideo) {
+			this.offScreenVideo.pause();
+			this.offScreenVideo.srcObject = null;
+			this.offScreenVideo = null;
+		}
+
+		if (this.offScreenCanvas) {
+			this.offScreenCanvas = null;
+			this.canvasCtx = null;
+		}
+
+		if (this._maskCanvas) {
+			this._maskCanvas = null;
+			this._maskCtx = null;
+		}
+
+		if (this._refinedMaskCanvas) {
+			this._refinedMaskCanvas = null;
+			this._refinedMaskCtx = null;
+		}
+
+		if (this._foregroundCanvas) {
+			this._foregroundCanvas = null;
+			this._foregroundCtx = null;
+		}
+
+		if (this._backgroundCanvas) {
+			this._backgroundCanvas = null;
+			this._backgroundCtx = null;
+		}
+
+		this.backgroundImage = null;
+
+		this.logger.info('VideoProcessor cleaned up');
+	}
+
+	/**
+	 * Dispose of all resources including the model
+	 */
+	async dispose() {
+		await this.cleanup();
+
+		if (this.selfieSegmentation) {
+			this.selfieSegmentation.close();
+			this.selfieSegmentation = null;
+		}
+
+		this.isInitialized = false;
+		this.logger.info('VideoProcessor disposed');
+	}
+
+	// Private methods
+
+	async _loadImage(url) {
+		return new Promise((resolve, reject) => {
+			const img = new Image();
+			img.crossOrigin = 'anonymous';
+			img.onload = () => resolve(img);
+			img.onerror = reject;
+			img.src = url;
+		});
+	}
+
+	async _startProcessingAndWaitForFirstFrame(type, blurLevel) {
+		return new Promise((resolve) => {
+			let firstFrameDrawn = false;
+
+			this.selfieSegmentation.onResults((results) => {
+				this._drawFrame(results, type, blurLevel);
+
+				if (!firstFrameDrawn) {
+					firstFrameDrawn = true;
+					this.logger.info('First frame processed and drawn');
+					resolve(true);
+				}
+			});
+
+			const processFrame = async () => {
+				if (!this.processing || !this.offScreenVideo) {
+					if (!firstFrameDrawn) resolve(false);
+					return;
+				}
+
+				// Check if video is ready and has dimensions
+				if (
+					this.offScreenVideo.readyState >= 2 &&
+					this.offScreenVideo.videoWidth > 0 &&
+					this.offScreenVideo.videoHeight > 0
+				) {
+					try {
+						await this.selfieSegmentation.send({ image: this.offScreenVideo });
+					} catch (err) {
+						this.logger.error('Error sending frame to MediaPipe:', err);
+					}
+				}
+
+				this.animationFrameId = requestAnimationFrame(processFrame);
+			};
+
+			// Timeout after 5 seconds if first frame never comes
+			setTimeout(() => {
+				if (!firstFrameDrawn) {
+					this.logger.warn('Timeout waiting for first frame');
+					resolve(false);
+				}
+			}, 5000);
+
+			processFrame();
+		});
+	}
+
+	_drawFrame(results, type, blurLevel) {
+		if (!this.canvasCtx || !this.offScreenCanvas) return;
+
+		const { width, height} = this.offScreenCanvas;
+
+		this.canvasCtx.save();
+		this.canvasCtx.clearRect(0, 0, width, height);
+
+		// Create temporary canvases for processing
+		if (!this._maskCanvas) {
+			this._maskCanvas = document.createElement('canvas');
+			this._maskCtx = this._maskCanvas.getContext('2d');
+		}
+		if (!this._refinedMaskCanvas) {
+			this._refinedMaskCanvas = document.createElement('canvas');
+			this._refinedMaskCtx = this._refinedMaskCanvas.getContext('2d');
+		}
+		if (!this._foregroundCanvas) {
+			this._foregroundCanvas = document.createElement('canvas');
+			this._foregroundCtx = this._foregroundCanvas.getContext('2d');
+		}
+		if (!this._backgroundCanvas) {
+			this._backgroundCanvas = document.createElement('canvas');
+			this._backgroundCtx = this._backgroundCanvas.getContext('2d');
+		}
+
+		// Set canvas sizes
+		this._maskCanvas.width = width;
+		this._maskCanvas.height = height;
+		this._refinedMaskCanvas.width = width;
+		this._refinedMaskCanvas.height = height;
+		this._foregroundCanvas.width = width;
+		this._foregroundCanvas.height = height;
+		this._backgroundCanvas.width = width;
+		this._backgroundCanvas.height = height;
+
+		// ===== STEP 1: Multi-pass mask refinement =====
+		// Draw raw mask
+		this._maskCtx.clearRect(0, 0, width, height);
+		this._maskCtx.drawImage(results.segmentationMask, 0, 0, width, height);
+
+		// Apply progressive blur for smooth edges
+		this._refinedMaskCtx.clearRect(0, 0, width, height);
+		// First pass: moderate blur
+		this._refinedMaskCtx.filter = 'blur(2px)';
+		this._refinedMaskCtx.drawImage(this._maskCanvas, 0, 0);
+		this._refinedMaskCtx.filter = 'none';
+
+		// Second pass: additional blur on top (cumulative effect)
+		this._refinedMaskCtx.globalAlpha = 1.0;
+		this._refinedMaskCtx.filter = `blur(${this.edgeBlurAmount}px)`;
+		this._refinedMaskCtx.drawImage(this._maskCanvas, 0, 0);
+		this._refinedMaskCtx.filter = 'none';
+		this._refinedMaskCtx.globalAlpha = 1.0;
+
+		// ===== STEP 2: Prepare background =====
+		this._backgroundCtx.clearRect(0, 0, width, height);
+		if (type === 'image' && this.backgroundImage) {
+			this._backgroundCtx.drawImage(this.backgroundImage, 0, 0, width, height);
+		} else if (type === 'blur') {
+			this._backgroundCtx.filter = `blur(${blurLevel}px)`;
+			this._backgroundCtx.drawImage(this.offScreenVideo, 0, 0, width, height);
+			this._backgroundCtx.filter = 'none';
+		}
+
+		// ===== STEP 3: Apply light wrapping =====
+		// Create a blurred version of background for light wrap
+		const lightWrapCanvas = document.createElement('canvas');
+		lightWrapCanvas.width = width;
+		lightWrapCanvas.height = height;
+		const lightWrapCtx = lightWrapCanvas.getContext('2d');
+
+		// Heavy blur on background for light wrap effect
+		lightWrapCtx.filter = 'blur(15px)';
+		lightWrapCtx.drawImage(this._backgroundCanvas, 0, 0);
+		lightWrapCtx.filter = 'none';
+
+		// Create inverted edge mask (for light wrap on edges only)
+		lightWrapCtx.globalCompositeOperation = 'destination-in';
+		lightWrapCtx.globalAlpha = this.lightWrapIntensity;
+
+		// Draw inverted mask to only affect edges
+		lightWrapCtx.filter = 'blur(8px) contrast(2)';
+		lightWrapCtx.drawImage(this._refinedMaskCanvas, 0, 0);
+		lightWrapCtx.filter = 'none';
+		lightWrapCtx.globalAlpha = 1.0;
+
+		// ===== STEP 4: Composite foreground =====
+		this._foregroundCtx.clearRect(0, 0, width, height);
+
+		// Draw person
+		this._foregroundCtx.drawImage(this.offScreenVideo, 0, 0, width, height);
+
+		// Apply refined mask using destination-in
+		this._foregroundCtx.globalCompositeOperation = 'destination-in';
+		this._foregroundCtx.drawImage(this._refinedMaskCanvas, 0, 0);
+		this._foregroundCtx.globalCompositeOperation = 'source-over';
+
+		// Add light wrap on foreground edges using screen blend
+		this._foregroundCtx.globalCompositeOperation = 'screen';
+		this._foregroundCtx.globalAlpha = 0.6;
+		this._foregroundCtx.drawImage(lightWrapCanvas, 0, 0);
+		this._foregroundCtx.globalAlpha = 1.0;
+		this._foregroundCtx.globalCompositeOperation = 'source-over';
+
+		// ===== STEP 5: Final composite =====
+		// Draw background
+		this.canvasCtx.drawImage(this._backgroundCanvas, 0, 0);
+
+		// Draw foreground with light wrap on top
+		this.canvasCtx.drawImage(this._foregroundCanvas, 0, 0);
+
+		this.canvasCtx.restore();
+	}
+}

package/index.js

@@ -0,0 +1,42 @@
+/**
+ * Video Meeting SDK - Main Entry Point
+ *
+ * @example
+ * import { VideoMeetingClient } from '@unboundcx/video-sdk';
+ *
+ * const client = new VideoMeetingClient({
+ *   serverUrl: 'wss://video.example.com',
+ *   debug: true
+ * });
+ *
+ * await client.connect(authToken);
+ * await client.joinRoom(roomId);
+ * await client.publishCamera();
+ */
+
+// Main client
+export { VideoMeetingClient } from './VideoMeetingClient.js';
+export { VideoProcessor } from './VideoProcessor.js';
+
+// Managers (for advanced usage)
+export { ConnectionManager } from './managers/ConnectionManager.js';
+export { MediasoupManager } from './managers/MediasoupManager.js';
+export { LocalMediaManager } from './managers/LocalMediaManager.js';
+export { RemoteMediaManager } from './managers/RemoteMediaManager.js';
+
+// Utilities
+export { EventEmitter } from './utils/EventEmitter.js';
+export { Logger } from './utils/Logger.js';
+
+// Error classes
+export {
+	SDKError,
+	ConnectionError,
+	MediasoupError, // Keep for backward compatibility, but also export as MediaTransportError
+	MediasoupError as MediaTransportError,
+	PermissionDeniedError,
+	DeviceNotFoundError,
+	StateError,
+	RoomError,
+	TimeoutError,
+} from './utils/errors.js';

package/managers/ConnectionManager.js

@@ -0,0 +1,243 @@
+import { io } from 'socket.io-client';
+import { EventEmitter } from '../utils/EventEmitter.js';
+import { Logger } from '../utils/Logger.js';
+import { ConnectionError, TimeoutError } from '../utils/errors.js';
+
+/**
+ * Manages Socket.io connection and signaling with the video server
+ *
+ * Events:
+ * - 'connected' - Connected to server
+ * - 'disconnected' - Disconnected from server
+ * - 'error' - Connection error
+ * - 'message' - Received message from server
+ */
+export class ConnectionManager extends EventEmitter {
+	/**
+	 * @param {Object} options
+	 * @param {string} options.serverUrl - WebSocket server URL
+	 * @param {boolean} options.debug - Enable debug logging
+	 */
+	constructor(options) {
+		super();
+
+		this.serverUrl = options.serverUrl;
+		this.logger = new Logger('SDK:ConnectionManager', options.debug);
+		this.socket = null;
+		this.isConnected = false;
+		this.reconnectAttempts = 0;
+		this.maxReconnectAttempts = 5;
+	}
+
+	/**
+	 * Connect to the server
+	 * @param {Object} auth - Authentication data
+	 * @returns {Promise<void>}
+	 */
+	async connect(auth = {}) {
+		if (this.isConnected) {
+			this.logger.warn('Already connected');
+			return;
+		}
+
+		this.logger.info('Connecting to server:', this.serverUrl);
+
+		return new Promise((resolve, reject) => {
+			try {
+				// Match the old working videoCreateSocket.js pattern exactly
+				// Keep it simple: just withCredentials and auth
+				const socketOptions = {
+					withCredentials: true,
+					auth,
+				};
+
+				console.log('ConnectionManager :: Auth object being sent:', auth);
+				console.log('ConnectionManager :: Full socket options:', socketOptions);
+
+				this.logger.info('Creating socket.io connection with options:', {
+					serverUrl: this.serverUrl,
+					...socketOptions
+				});
+
+				this.socket = io(this.serverUrl, socketOptions);
+
+				this.logger.info('Socket.io instance created, waiting for connect event');
+
+				// Setup event listeners
+				this.socket.on('connect', () => {
+					this.logger.info('Connected to server');
+					this.isConnected = true;
+					this.reconnectAttempts = 0;
+					this.emit('connected', { socketId: this.socket.id });
+					resolve();
+				});
+
+				this.socket.on('disconnect', (reason) => {
+					this.logger.warn('Disconnected from server:', reason);
+					this.isConnected = false;
+					this.emit('disconnected', { reason });
+				});
+
+				this.socket.on('connect_error', (error) => {
+					this.logger.error('Connection error:', error);
+					this.reconnectAttempts++;
+
+					const connectionError = new ConnectionError(
+						'Failed to connect to server',
+						{ error: error.message, attempts: this.reconnectAttempts }
+					);
+
+					this.emit('error', connectionError);
+
+					if (this.reconnectAttempts >= this.maxReconnectAttempts) {
+						reject(connectionError);
+					}
+				});
+
+				// Timeout after 10 seconds
+				const timeout = setTimeout(() => {
+					if (!this.isConnected) {
+						this.socket?.disconnect();
+						reject(new TimeoutError('connect', 10000));
+					}
+				}, 10000);
+
+				this.socket.once('connect', () => clearTimeout(timeout));
+
+			} catch (error) {
+				this.logger.error('Failed to create socket:', error);
+				reject(new ConnectionError('Failed to create socket connection', { error }));
+			}
+		});
+	}
+
+	/**
+	 * Disconnect from server
+	 * @returns {Promise<void>}
+	 */
+	async disconnect() {
+		if (!this.socket) {
+			this.logger.warn('No socket to disconnect');
+			return;
+		}
+
+		this.logger.info('Disconnecting from server');
+
+		return new Promise((resolve) => {
+			this.socket.once('disconnect', () => {
+				this.isConnected = false;
+				this.socket = null;
+				resolve();
+			});
+
+			this.socket.disconnect();
+
+			// Force resolve after 1 second
+			setTimeout(resolve, 1000);
+		});
+	}
+
+	/**
+	 * Emit an event to the server and wait for response
+	 * @param {string} event - Event name
+	 * @param {*} data - Event data
+	 * @param {number} timeout - Timeout in ms (default 5000)
+	 * @returns {Promise<*>} Server response
+	 */
+	async request(event, data = {}, timeout = 5000) {
+		if (!this.isConnected || !this.socket) {
+			throw new ConnectionError('Not connected to server');
+		}
+
+		this.logger.log('Request:', event, data);
+
+		return new Promise((resolve, reject) => {
+			const timer = setTimeout(() => {
+				reject(new TimeoutError(event, timeout));
+			}, timeout);
+
+			this.socket.emit(event, data, (response) => {
+				clearTimeout(timer);
+
+				if (response?.error) {
+					this.logger.error('Request error:', event, response.error);
+					reject(new ConnectionError(
+						`Server error: ${response.error.message || 'Unknown error'}`,
+						{ event, error: response.error }
+					));
+				} else {
+					this.logger.log('Response:', event, response);
+					resolve(response);
+				}
+			});
+		});
+	}
+
+	/**
+	 * Emit an event to server without waiting for response
+	 * @param {string} event - Event name
+	 * @param {*} data - Event data
+	 */
+	emit(event, data = {}) {
+		if (!this.isConnected || !this.socket) {
+			throw new ConnectionError('Not connected to server');
+		}
+
+		this.logger.log('Emit:', event, data);
+		this.socket.emit(event, data);
+	}
+
+	/**
+	 * Alias for emit (backward compatibility)
+	 * @param {string} event - Event name
+	 * @param {*} data - Event data
+	 */
+	send(event, data = {}) {
+		return this.emit(event, data);
+	}
+
+	/**
+	 * Listen for server events
+	 * @param {string} event - Event name
+	 * @param {Function} handler - Event handler
+	 * @returns {Function} Unsubscribe function
+	 */
+	onServerEvent(event, handler) {
+		if (!this.socket) {
+			throw new ConnectionError('Socket not initialized');
+		}
+
+		this.logger.log('Listening for server event:', event);
+		this.socket.on(event, handler);
+
+		// Return unsubscribe function
+		return () => {
+			this.socket?.off(event, handler);
+		};
+	}
+
+	/**
+	 * Remove server event listener
+	 * @param {string} event - Event name
+	 * @param {Function} handler - Event handler to remove
+	 */
+	offServerEvent(event, handler) {
+		this.socket?.off(event, handler);
+	}
+
+	/**
+	 * Get connection state
+	 * @returns {boolean}
+	 */
+	get connected() {
+		return this.isConnected && this.socket?.connected;
+	}
+
+	/**
+	 * Get socket ID
+	 * @returns {string|null}
+	 */
+	get socketId() {
+		return this.socket?.id || null;
+	}
+}