@unboundcx/video-sdk-client 2.0.2 → 2.0.4

package/VideoMeetingClient.js

@@ -206,6 +206,14 @@ export class VideoMeetingClient extends EventEmitter {
       this.emit("device:changed", data);
     });
 
+    this.localMedia.on("camera:degraded", (data) => {
+      this.emit("camera:degraded", data);
+    });
+
+    this.localMedia.on("camera:restarted", (data) => {
+      this.emit("camera:restarted", data);
+    });
+
     // Remote media events
     this.remoteMedia.on("participant:added", (data) => {
       this.emit("participant:joined", data);
@@ -882,6 +890,21 @@ export class VideoMeetingClient extends EventEmitter {
     return await this.localMedia.updateCameraBackground(options);
   }
 
+  /**
+   * Restart the local camera in place. Use after a 'camera:degraded'
+   * event, or as a user-triggered recovery from a frozen video state.
+   * Re-acquires the camera, reapplies any active background, and
+   * replaces the track on the existing producer (no renegotiation).
+   *
+   * @param {Object} [options] - Optional deviceId/facingMode override.
+   *                             If omitted, reuses the last-known camera source.
+   * @returns {Promise<MediaStream>}
+   */
+  async restartCamera(options) {
+    this._ensureInRoom();
+    return await this.localMedia.restartCamera(options);
+  }
+
   /**
    * Mute camera
    * @returns {Promise<void>}

package/managers/LocalMediaManager.js

@@ -57,6 +57,15 @@ export class LocalMediaManager extends EventEmitter {
 		// Video processor for virtual backgrounds
 		this.videoProcessor = new VideoProcessor({ debug: options.debug });
 		this.currentBackgroundOptions = null;
+
+		// When a virtual background is active, `streams.camera` is the
+		// canvas-captured (processed) stream — its track has no deviceId
+		// or facingMode. We keep the *raw* camera stream and the original
+		// source descriptor around so subsequent operations (background
+		// swaps, camera changes) can re-acquire or reuse the real camera
+		// rather than reading constraints off the canvas track.
+		this.rawCameraStream = null;
+		this.cameraSource = null; // { deviceId } | { facingMode } | null
 	}
 
 	/**
@@ -186,6 +195,19 @@ export class LocalMediaManager extends EventEmitter {
 				}
 			}
 
+			// Remember the raw camera stream + source descriptor BEFORE any
+			// background processing wraps it. Used by updateCameraBackground
+			// and changeCamera so they don't read constraints off a canvas
+			// track (which has no deviceId / facingMode).
+			this.rawCameraStream = stream;
+			if (options.facingMode) {
+				this.cameraSource = { facingMode: options.facingMode };
+			} else {
+				const trackDeviceId =
+					originalTrack.getSettings().deviceId || options.deviceId || null;
+				this.cameraSource = trackDeviceId ? { deviceId: trackDeviceId } : null;
+			}
+
 			// Apply background effect if specified
 			if (options.background && options.background.type !== 'none') {
 				this.logger.info('Applying background effect:', options.background);
@@ -207,12 +229,11 @@ export class LocalMediaManager extends EventEmitter {
 			// Store stream
 			this.streams.camera = stream;
 
-			// Setup track ended handler
-			track.addEventListener('ended', () => {
-				this.logger.warn('Camera track ended');
-				this.emit('track:ended', { type: 'camera' });
-				this.streams.camera = null;
-			});
+			// Attach health watchers (ended / muted) on the producer's
+			// current track. _watchCameraTrack handles both ended and
+			// browser-side mute (which is distinct from user mute and
+			// indicates the OS took the camera, USB unplug, etc.).
+			this._watchCameraTrack(track);
 
 			// Publish to mediasoup
 			const producer = await this.mediasoup.produce(track, {
@@ -442,6 +463,22 @@ export class LocalMediaManager extends EventEmitter {
 		// Stop all tracks
 		this.streams.camera.getTracks().forEach((track) => track.stop());
 
+		// Also stop the raw camera stream if it's distinct (background was
+		// active — processed/canvas stream wraps the raw camera stream).
+		if (this.rawCameraStream && this.rawCameraStream !== this.streams.camera) {
+			this.rawCameraStream.getTracks().forEach((t) => t.stop());
+		}
+
+		// Tear down processor if a background was active.
+		if (this.currentBackgroundOptions) {
+			try {
+				await this.videoProcessor.cleanup();
+			} catch (cleanupErr) {
+				this.logger.warn('Processor cleanup during stopCamera failed:', cleanupErr);
+			}
+			this.currentBackgroundOptions = null;
+		}
+
 		// Close producer
 		if (this.producers.video) {
 			await this.mediasoup.closeProducer('video');
@@ -449,6 +486,8 @@ export class LocalMediaManager extends EventEmitter {
 		}
 
 		this.streams.camera = null;
+		this.rawCameraStream = null;
+		this.cameraSource = null;
 
 		this.emit('stream:removed', { type: 'camera' });
 	}
@@ -687,18 +726,28 @@ export class LocalMediaManager extends EventEmitter {
 			throw new Error('No active camera to change');
 		}
 
+		const oldProcessedStream = this.streams.camera;
+		const oldRawStream = this.rawCameraStream;
+
 		try {
-			// Get new stream
+			// Get new stream (always fresh — camera device is changing).
 			const constraints = this._getVideoConstraints(options);
-			let stream = await navigator.mediaDevices.getUserMedia({
+			const rawStream = await navigator.mediaDevices.getUserMedia({
 				video: constraints,
 			});
 
+			let stream = rawStream;
+
 			// Reapply background effect if it was active
 			if (this.currentBackgroundOptions) {
 				this.logger.info('Reapplying background effect after camera change');
+				try {
+					await this.videoProcessor.cleanup();
+				} catch (cleanupErr) {
+					this.logger.warn('Processor cleanup during camera change failed:', cleanupErr);
+				}
 				stream = await this.videoProcessor.applyBackground(
-					stream,
+					rawStream,
 					this.currentBackgroundOptions,
 				);
 			}
@@ -707,6 +756,7 @@ export class LocalMediaManager extends EventEmitter {
 
 			// Replace track in producer
 			await this.producers.video.replaceTrack({ track: newTrack });
+			this._watchCameraTrack(newTrack);
 
 			// If producer was paused (camera muted), resume it
 			// When user manually changes camera, they likely want to use it
@@ -716,13 +766,32 @@ export class LocalMediaManager extends EventEmitter {
 				this.muteState.camera = false;
 			}
 
-			// Stop old stream
-			if (this.streams.camera) {
-				this.streams.camera.getTracks().forEach((track) => track.stop());
+			// Stop the OLD raw camera stream (the actual device). The old
+			// processed/canvas stream's track is no longer attached to the
+			// producer; stop it too so the canvas doesn't keep pumping.
+			if (oldRawStream && oldRawStream !== rawStream) {
+				oldRawStream.getTracks().forEach((t) => t.stop());
+			}
+			if (
+				oldProcessedStream &&
+				oldProcessedStream !== oldRawStream &&
+				oldProcessedStream !== stream
+			) {
+				oldProcessedStream.getTracks().forEach((t) => t.stop());
 			}
 
-			// Store new stream
+			// Store new streams + source descriptor.
 			this.streams.camera = stream;
+			this.rawCameraStream = rawStream;
+			if (options.facingMode) {
+				this.cameraSource = { facingMode: options.facingMode };
+			} else {
+				const trackDeviceId =
+					rawStream.getVideoTracks()[0]?.getSettings().deviceId ||
+					options.deviceId ||
+					null;
+				this.cameraSource = trackDeviceId ? { deviceId: trackDeviceId } : null;
+			}
 
 			this.logger.info('Camera changed successfully');
 
@@ -766,66 +835,129 @@ export class LocalMediaManager extends EventEmitter {
 			throw new Error('No active video producer');
 		}
 
-		try {
-			// Clean up old processor if exists
-			if (this.currentBackgroundOptions) {
-				await this.videoProcessor.cleanup();
-			}
-
-			// Get current device ID
-			const currentTrack = this.streams.camera.getVideoTracks()[0];
-			const deviceId = currentTrack.getSettings().deviceId;
+		// Remember everything we'd need to restore on failure. We do NOT
+		// stop the existing canvas / camera or tear down the processor
+		// until the NEW track has been swapped in successfully — otherwise
+		// a failure mid-swap leaves the producer holding a stopped track
+		// and the user's video freezes locally and on remotes.
+		const hadBackground = !!this.currentBackgroundOptions;
+		const oldProcessedStream = this.streams.camera;
+		const oldRawStream = this.rawCameraStream;
 
-			// If removing background, restart with original stream
+		try {
+			// REMOVE BACKGROUND
 			if (options.type === 'none') {
 				this.logger.info('Removing background effect');
-				this.currentBackgroundOptions = null;
 
-				const constraints = this._getVideoConstraints({ deviceId });
-				const newStream = await navigator.mediaDevices.getUserMedia({
-					video: constraints,
-				});
+				// Prefer the raw camera stream we already have — its track is
+				// the real device track. If it's somehow gone, fall back to
+				// getUserMedia using the recorded source.
+				let newStream = null;
+				const rawTrack = oldRawStream?.getVideoTracks?.()[0];
+				if (rawTrack && rawTrack.readyState === 'live') {
+					newStream = oldRawStream;
+				} else {
+					const constraints = this._getVideoConstraints(
+						this.cameraSource || {},
+					);
+					newStream = await navigator.mediaDevices.getUserMedia({
+						video: constraints,
+					});
+				}
 
 				const newTrack = newStream.getVideoTracks()[0];
-
 				await this.producers.video.replaceTrack({ track: newTrack });
+				this._watchCameraTrack(newTrack);
 
-				this.streams.camera.getTracks().forEach((t) => t.stop());
+				// Commit new state, then tear down old.
 				this.streams.camera = newStream;
+				this.rawCameraStream = newStream;
+				this.currentBackgroundOptions = null;
+
+				if (hadBackground) {
+					try {
+						await this.videoProcessor.cleanup();
+					} catch (cleanupErr) {
+						this.logger.warn('Processor cleanup after remove failed:', cleanupErr);
+					}
+					// Stop the processed (canvas) stream's track if it's
+					// distinct from the raw stream we just promoted.
+					if (oldProcessedStream && oldProcessedStream !== newStream) {
+						oldProcessedStream.getTracks().forEach((t) => t.stop());
+					}
+				}
 
 				this.logger.info('Background effect removed');
 				this.emit('background:changed', { type: 'none' });
-
 				return newStream;
 			}
 
-			// Apply new background effect
-			this.currentBackgroundOptions = options;
-
-			// Get fresh stream
-			const constraints = this._getVideoConstraints({ deviceId });
-			let stream = await navigator.mediaDevices.getUserMedia({
-				video: constraints,
-			});
+			// APPLY / SWAP BACKGROUND
+			//
+			// Reuse the raw camera stream if it's still live — that's the
+			// common case for switching between backgrounds without doing
+			// a second getUserMedia (which is slow and can hit
+			// OverconstrainedError because the canvas track has no
+			// deviceId).
+			let rawStream = oldRawStream;
+			const rawTrack = rawStream?.getVideoTracks?.()[0];
+			const rawAlive = !!rawTrack && rawTrack.readyState === 'live';
+
+			if (!rawAlive) {
+				const constraints = this._getVideoConstraints(
+					this.cameraSource || {},
+				);
+				rawStream = await navigator.mediaDevices.getUserMedia({
+					video: constraints,
+				});
+			}
 
-			// Apply background
-			stream = await this.videoProcessor.applyBackground(stream, options);
+			// If swapping (background already active) we must tear down the
+			// processor before starting a new one — its canvas / offscreen
+			// video are single-instance state.
+			if (hadBackground) {
+				try {
+					await this.videoProcessor.cleanup();
+				} catch (cleanupErr) {
+					this.logger.warn('Processor cleanup before swap failed:', cleanupErr);
+				}
+			}
 
-			const newTrack = stream.getVideoTracks()[0];
+			const processedStream = await this.videoProcessor.applyBackground(
+				rawStream,
+				options,
+			);
+			const newTrack = processedStream.getVideoTracks()[0];
 
-			// Replace track in producer
 			await this.producers.video.replaceTrack({ track: newTrack });
+			this._watchCameraTrack(newTrack);
 
-			// Stop old stream
-			this.streams.camera.getTracks().forEach((t) => t.stop());
-			this.streams.camera = stream;
+			// Commit new state.
+			this.currentBackgroundOptions = options;
+			this.streams.camera = processedStream;
+			this.rawCameraStream = rawStream;
+
+			// Stop the OLD processed canvas stream's track (it was wrapping
+			// the processor we just cleaned up). The raw stream is
+			// intentionally preserved — it's still feeding the new processor.
+			if (
+				hadBackground &&
+				oldProcessedStream &&
+				oldProcessedStream !== processedStream &&
+				oldProcessedStream !== rawStream
+			) {
+				oldProcessedStream.getTracks().forEach((t) => t.stop());
+			}
 
 			this.logger.info('Background effect updated successfully');
 			this.emit('background:changed', options);
 
-			return stream;
+			return processedStream;
 		} catch (error) {
 			this.logger.error('Failed to update camera background:', error);
+			// We did not mutate streams.camera or stop the old track until
+			// after the swap succeeded, so the producer is still holding a
+			// live track and the user's video keeps flowing.
 			throw wrapError(error, 'updateCameraBackground');
 		}
 	}
@@ -1036,6 +1168,165 @@ export class LocalMediaManager extends EventEmitter {
 		return this.muteState.microphone;
 	}
 
+	/**
+	 * Attach health watchers to the producer's current video track.
+	 *
+	 * Emits:
+	 *   - 'track:ended' { type: 'camera' }   — kept for back-compat
+	 *   - 'camera:degraded' { reason, track } — track is no longer
+	 *       delivering frames (ended, OS-level muted, etc.). Remotes
+	 *       will see a frozen frame until the producer is paused or the
+	 *       track is replaced via restartCamera().
+	 *
+	 * The 'mute' event here is the browser's own muting (camera taken
+	 * over by another app, USB unplug, OS privacy switch) — distinct
+	 * from the user pressing our mute button.
+	 */
+	_watchCameraTrack(track) {
+		if (!track) return;
+		// Stash so subsequent watchers can replace listeners cleanly.
+		if (this._watchedCameraTrack === track) return;
+		this._watchedCameraTrack = track;
+
+		const onEnded = () => {
+			this.logger.warn('Camera track ended');
+			this.emit('track:ended', { type: 'camera' });
+			if (this.streams.camera) this.streams.camera = null;
+			this.emit('camera:degraded', { reason: 'ended', track });
+		};
+		const onMute = () => {
+			this.logger.warn('Camera track muted (browser-side)');
+			this.emit('camera:degraded', { reason: 'muted', track });
+		};
+
+		track.addEventListener('ended', onEnded);
+		track.addEventListener('mute', onMute);
+	}
+
+	/**
+	 * Restart the local camera in place.
+	 *
+	 * Re-acquires the camera via `cameraSource` (or any caller-supplied
+	 * deviceId/facingMode), reapplies the current background if one was
+	 * active, and `replaceTrack`s the live producer. Use this when
+	 * 'camera:degraded' has fired or the user manually clicks "Restart
+	 * camera" after a freeze.
+	 *
+	 * Does NOT change isCameraMuted. If the producer was paused on
+	 * degradation, the caller is responsible for resuming it once the
+	 * new track is in place (we resume here automatically for ergonomics).
+	 *
+	 * @param {Object} [options] - Optional deviceId/facingMode override.
+	 * @returns {Promise<MediaStream>} New camera stream.
+	 */
+	async restartCamera(options = {}) {
+		this.logger.info('Restarting camera', options);
+
+		if (!this.producers.video) {
+			throw new Error('No active camera producer to restart');
+		}
+
+		const source =
+			Object.keys(options).length > 0 ? options : this.cameraSource || {};
+
+		const oldProcessedStream = this.streams.camera;
+		const oldRawStream = this.rawCameraStream;
+
+		try {
+			const constraints = this._getVideoConstraints(source);
+			let rawStream;
+			try {
+				rawStream = await navigator.mediaDevices.getUserMedia({
+					video: constraints,
+				});
+			} catch (err) {
+				if (
+					err.name === 'OverconstrainedError' ||
+					err.name === 'ConstraintNotSatisfiedError'
+				) {
+					this.logger.warn(
+						'Restart hit constraints, retrying with ideal-only:',
+						err.constraint,
+					);
+					const fallback = {
+						...(source.deviceId ? { deviceId: { exact: source.deviceId } } : {}),
+						...(source.facingMode
+							? { facingMode: { exact: source.facingMode } }
+							: {}),
+						width: { ideal: constraints.width?.ideal },
+						height: { ideal: constraints.height?.ideal },
+						frameRate: constraints.frameRate,
+					};
+					rawStream = await navigator.mediaDevices.getUserMedia({
+						video: fallback,
+					});
+				} else {
+					throw err;
+				}
+			}
+
+			let stream = rawStream;
+			if (this.currentBackgroundOptions) {
+				try {
+					await this.videoProcessor.cleanup();
+				} catch (cleanupErr) {
+					this.logger.warn('Processor cleanup during restart failed:', cleanupErr);
+				}
+				stream = await this.videoProcessor.applyBackground(
+					rawStream,
+					this.currentBackgroundOptions,
+				);
+			}
+
+			const newTrack = stream.getVideoTracks()[0];
+			await this.producers.video.replaceTrack({ track: newTrack });
+
+			// If we paused the producer when degradation was detected,
+			// resume now that a healthy track is attached. Don't touch
+			// the user's mute state.
+			if (this.producers.video.paused && !this.muteState.camera) {
+				try {
+					this.producers.video.resume();
+				} catch (resumeErr) {
+					this.logger.warn('Producer resume after restart failed:', resumeErr);
+				}
+			}
+
+			// Commit + stop old streams.
+			this.streams.camera = stream;
+			this.rawCameraStream = rawStream;
+			if (source.facingMode) {
+				this.cameraSource = { facingMode: source.facingMode };
+			} else {
+				const trackDeviceId =
+					rawStream.getVideoTracks()[0]?.getSettings().deviceId ||
+					source.deviceId ||
+					null;
+				this.cameraSource = trackDeviceId ? { deviceId: trackDeviceId } : null;
+			}
+
+			if (oldRawStream && oldRawStream !== rawStream) {
+				oldRawStream.getTracks().forEach((t) => t.stop());
+			}
+			if (
+				oldProcessedStream &&
+				oldProcessedStream !== oldRawStream &&
+				oldProcessedStream !== stream
+			) {
+				oldProcessedStream.getTracks().forEach((t) => t.stop());
+			}
+
+			this._watchCameraTrack(newTrack);
+			this.emit('camera:restarted', { stream });
+			this.logger.info('Camera restarted successfully');
+
+			return stream;
+		} catch (error) {
+			this.logger.error('Failed to restart camera:', error);
+			throw wrapError(error, 'restartCamera');
+		}
+	}
+
 	/**
 	 * Clean up all local streams
 	 */

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@unboundcx/video-sdk-client",
-	"version": "2.0.2",
+	"version": "2.0.4",
 	"description": "Framework-agnostic WebRTC video meeting SDK powered by mediasoup",
 	"type": "module",
 	"main": "index.js",