@phystack/hub-client 4.5.19-dev → 4.5.21-dev

package/src/services/webrtc/media-stream-handler.ts

@@ -0,0 +1,536 @@
+/**
+ * MediaStreamHandler
+ *
+ * Manages MediaStream connections including:
+ * - Stream setup for sending/receiving media
+ * - Track lifecycle management (add/remove)
+ * - Frame activity monitoring for stale connection detection
+ * - Persistent stream abstraction across reconnections
+ */
+
+import { PeerConnectionManager, PeerConnectionManagerOptions } from './peer-connection-manager';
+import {
+  TwinMessagingInterface,
+  PhygridMediaStream,
+  MediaStreamOptions,
+  PeerConnectionConfig,
+  ExtendedMediaStreamTrack,
+  DEFAULT_WEBRTC_OPTIONS,
+} from './types';
+
+const FRAME_INACTIVITY_TIMEOUT = 5000; // 5 seconds without frames = stale
+
+export interface MediaStreamHandlerOptions extends Partial<PeerConnectionManagerOptions> {
+  /** Use STUN servers. Default: true */
+  useStun?: boolean;
+
+  /** STUN server URLs */
+  stunServers?: string[];
+
+  /** Media options */
+  mediaOptions?: MediaStreamOptions;
+}
+
+export class MediaStreamHandler {
+  private targetTwinId: string;
+  private channelName: string;
+  private isInitiator: boolean;
+  private twinMessaging: TwinMessagingInterface;
+  private options: Required<Omit<MediaStreamHandlerOptions, 'mediaOptions'>> & {
+    mediaOptions: MediaStreamOptions;
+  };
+
+  private pcManager: PeerConnectionManager | null = null;
+  private remoteStream: MediaStream | null = null;
+  private localStream: MediaStream | null = null;
+  private isClosed = false;
+  private isReceiving = false;
+  private isConnecting = false;
+
+  // Track management
+  private tracks: MediaStreamTrack[] = [];
+  private trackLastFrameTime: Map<string, number> = new Map();
+  private frameActivityInterval: NodeJS.Timeout | null = null;
+
+  // Event listeners
+  private trackListeners: Set<(track: MediaStreamTrack) => void> = new Set();
+  private frameListeners: Set<(frameData: any) => void> = new Set();
+  private closeListeners: Set<() => void> = new Set();
+
+  // Callbacks for WebRTCManager
+  private onConnectedCallback?: () => void;
+  private onDisconnectedCallback?: () => void;
+  private onErrorCallback?: (error: Error) => void;
+  private onReconnectingCallback?: (attempt: number) => void;
+  private onReconnectedCallback?: (attempt: number) => void;
+
+  constructor(
+    targetTwinId: string,
+    isInitiator: boolean,
+    twinMessaging: TwinMessagingInterface,
+    options: MediaStreamHandlerOptions = {},
+    channelName: string = 'default'
+  ) {
+    this.targetTwinId = targetTwinId;
+    this.channelName = channelName;
+    this.isInitiator = isInitiator;
+    this.twinMessaging = twinMessaging;
+    this.options = {
+      useStun: options.useStun ?? DEFAULT_WEBRTC_OPTIONS.useStun,
+      stunServers: options.stunServers ?? DEFAULT_WEBRTC_OPTIONS.stunServers,
+      connectionTimeout: options.connectionTimeout ?? DEFAULT_WEBRTC_OPTIONS.connectionTimeout,
+      initialRetryDelay: options.initialRetryDelay ?? DEFAULT_WEBRTC_OPTIONS.initialRetryDelay,
+      maxRetryDelay: options.maxRetryDelay ?? DEFAULT_WEBRTC_OPTIONS.maxRetryDelay,
+      mediaOptions: {
+        direction: options.mediaOptions?.direction ?? (isInitiator ? 'recvonly' : 'sendrecv'),
+        localStream: options.mediaOptions?.localStream,
+      },
+    };
+
+    // Create local stream for non-initiator if not provided
+    if (!isInitiator && !this.options.mediaOptions.localStream) {
+      this.localStream = new MediaStream();
+      this.options.mediaOptions.localStream = this.localStream;
+    } else {
+      this.localStream = this.options.mediaOptions.localStream ?? null;
+    }
+  }
+
+  /**
+   * Set callbacks for connection events
+   */
+  setCallbacks(callbacks: {
+    onConnected?: () => void;
+    onDisconnected?: () => void;
+    onError?: (error: Error) => void;
+    onReconnecting?: (attempt: number) => void;
+    onReconnected?: (attempt: number) => void;
+  }): void {
+    this.onConnectedCallback = callbacks.onConnected;
+    this.onDisconnectedCallback = callbacks.onDisconnected;
+    this.onErrorCallback = callbacks.onError;
+    this.onReconnectingCallback = callbacks.onReconnecting;
+    this.onReconnectedCallback = callbacks.onReconnected;
+  }
+
+  /**
+   * Connect and return a PhygridMediaStream
+   */
+  async connect(): Promise<PhygridMediaStream> {
+    if (this.isClosed) {
+      throw new Error('MediaStreamHandler has been closed');
+    }
+
+    this.isConnecting = true;
+    // Channel prefix includes name for unique signaling per channel
+    const channelPrefix = `media-${this.channelName}`;
+
+    const pcConfig: PeerConnectionConfig = {
+      targetTwinId: this.targetTwinId,
+      isInitiator: this.isInitiator,
+      connectionType: 'mediastream',
+      channelPrefix,
+      useStun: this.options.useStun,
+      stunServers: this.options.stunServers,
+      onConnected: () => this.handlePeerConnected(),
+      onDisconnected: () => this.handlePeerDisconnected(),
+      onError: (err) => this.handleError(err),
+      onReconnecting: (attempt) => {
+        this.isConnecting = true;
+        this.isReceiving = false;
+        this.onReconnectingCallback?.(attempt);
+      },
+      onReconnected: (attempt) => this.handleReconnected(attempt),
+      // Setup transceivers/tracks BEFORE offer is created (critical for ICE gathering)
+      onPeerConnectionCreated: (pc) => {
+        if (this.isInitiator) {
+          if (this.localStream && this.localStream.getTracks().length > 0) {
+            // Initiator with local stream: Add actual tracks before offer
+            this.localStream.getTracks().forEach((track) => {
+              pc.addTrack(track, this.localStream!);
+            });
+          } else {
+            // Initiator without local stream: Add transceiver for receiving
+            pc.addTransceiver('video', { direction: this.options.mediaOptions.direction });
+          }
+        } else if (this.localStream) {
+          // Responder: Add local tracks before answer is sent
+          this.localStream.getTracks().forEach((track) => {
+            pc.addTrack(track, this.localStream!);
+          });
+        }
+
+        // Handle incoming tracks (both sides)
+        pc.ontrack = (event) => {
+          // Get the stream from the event or create one if not provided
+          // Some WebRTC implementations may not provide streams in the event
+          let stream: MediaStream;
+          if (event.streams && event.streams[0]) {
+            stream = event.streams[0];
+          } else {
+            // Create a stream if not provided (some WebRTC implementations don't include streams)
+            if (!this.remoteStream) {
+              this.remoteStream = new MediaStream();
+            }
+            stream = this.remoteStream;
+            // Add the track to our created stream
+            if (!stream.getTrackById(event.track.id)) {
+              stream.addTrack(event.track);
+            }
+          }
+
+          this.remoteStream = stream;
+          this.handleRemoteStream(stream);
+        };
+      },
+    };
+
+    this.pcManager = new PeerConnectionManager(pcConfig, this.twinMessaging, {
+      connectionTimeout: this.options.connectionTimeout,
+      initialRetryDelay: this.options.initialRetryDelay,
+      maxRetryDelay: this.options.maxRetryDelay,
+    });
+
+    const pc = await this.pcManager.connect();
+    await this.setupMediaStream(pc);
+
+    return this.createPhygridMediaStream();
+  }
+
+  /**
+   * Close the media stream and clean up
+   */
+  close(): void {
+    if (this.isClosed) return;
+
+    this.isClosed = true;
+    this.isReceiving = false;
+
+    this.stopFrameActivityMonitor();
+
+    // Stop all tracks
+    this.tracks.forEach((track) => {
+      try {
+        track.stop();
+      } catch (err) {
+        console.error('[MediaStreamHandler] Error stopping track:', err);
+      }
+    });
+    this.tracks = [];
+
+    if (this.remoteStream) {
+      this.remoteStream.getTracks().forEach((track) => track.stop());
+      this.remoteStream = null;
+    }
+
+    if (this.localStream) {
+      this.localStream.getTracks().forEach((track) => track.stop());
+      this.localStream = null;
+    }
+
+    if (this.pcManager) {
+      this.pcManager.close();
+      this.pcManager = null;
+    }
+
+    this.notifyCloseListeners();
+    this.trackListeners.clear();
+    this.frameListeners.clear();
+    this.closeListeners.clear();
+    this.trackLastFrameTime.clear();
+  }
+
+  // ===========================================================================
+  // Private Methods
+  // ===========================================================================
+
+  private async setupMediaStream(_pc: RTCPeerConnection): Promise<void> {
+    // Transceivers and ontrack handler are set up in onPeerConnectionCreated
+    // This method now just waits for the connection to be established
+    // or returns immediately if we're not expecting incoming tracks
+    if (this.options.mediaOptions.direction === 'sendonly') {
+      return;
+    }
+
+    // For recvonly/sendrecv, wait for tracks if not already received
+    if (this.remoteStream && this.remoteStream.getTracks().length > 0) {
+      return; // Already have tracks
+    }
+
+    // Wait a short time for tracks to arrive (they may come after connection)
+    return new Promise<void>((resolve) => {
+      const timeout = setTimeout(() => {
+        console.log('[MediaStreamHandler] Proceeding without waiting for remote tracks');
+        resolve();
+      }, 1000);
+
+      // If we get a track, resolve immediately
+      const checkForTracks = () => {
+        if (this.remoteStream && this.remoteStream.getTracks().length > 0) {
+          clearTimeout(timeout);
+          resolve();
+        }
+      };
+
+      // Check periodically
+      const interval = setInterval(() => {
+        checkForTracks();
+        if (this.remoteStream && this.remoteStream.getTracks().length > 0) {
+          clearInterval(interval);
+        }
+      }, 100);
+
+      // Clean up interval on timeout
+      setTimeout(() => clearInterval(interval), 1000);
+    });
+  }
+
+  private handleRemoteStream(stream: MediaStream): void {
+    this.startFrameActivityMonitor();
+
+    stream.getTracks().forEach((track) => {
+      this.addTrackToCollection(track, stream);
+    });
+
+    this.onConnectedCallback?.();
+  }
+
+  private addTrackToCollection(track: MediaStreamTrack, stream: MediaStream): void {
+    // Check for duplicate
+    const existingIndex = this.tracks.findIndex((existingTrack) => existingTrack.id === track.id);
+    if (existingIndex !== -1) {
+      this.tracks.splice(existingIndex, 1);
+    }
+
+    this.tracks.push(track);
+    this.trackLastFrameTime.set(track.id, Date.now());
+
+    // Set up frame monitoring for the track
+    this.setupTrackFrameMonitoring(track, stream);
+
+    // Notify listeners
+    this.notifyTrackListeners(track);
+  }
+
+  private setupTrackFrameMonitoring(track: MediaStreamTrack, stream: MediaStream): void {
+    const extendedTrack = track as ExtendedMediaStreamTrack;
+
+    // Try to use onFrame if available (Node.js @roamhq/wrtc)
+    if (typeof extendedTrack.onFrame === 'function') {
+      const originalOnFrame = extendedTrack.onFrame;
+      extendedTrack.onFrame = (frame: any) => {
+        this.trackLastFrameTime.set(track.id, Date.now());
+        this.isReceiving = true;
+        this.notifyFrameListeners(frame);
+        originalOnFrame.call(extendedTrack, frame);
+      };
+    }
+
+    // Handle track ended
+    track.onended = () => {
+      this.trackLastFrameTime.delete(track.id);
+      const index = this.tracks.findIndex((existingTrack) => existingTrack.id === track.id);
+      if (index !== -1) {
+        this.tracks.splice(index, 1);
+      }
+
+      // Check if all tracks ended
+      const allEnded = stream.getTracks().every((streamTrack) => streamTrack.readyState === 'ended');
+      if (allEnded && !this.isClosed) {
+        this.handlePeerDisconnected();
+      }
+    };
+  }
+
+  private startFrameActivityMonitor(): void {
+    if (this.frameActivityInterval) return;
+
+    this.frameActivityInterval = setInterval(() => {
+      this.checkFrameActivity();
+    }, 1000);
+  }
+
+  private stopFrameActivityMonitor(): void {
+    if (this.frameActivityInterval) {
+      clearInterval(this.frameActivityInterval);
+      this.frameActivityInterval = null;
+    }
+  }
+
+  private checkFrameActivity(): void {
+    if (this.trackLastFrameTime.size === 0) return;
+
+    const now = Date.now();
+    let hasActiveTrack = false;
+
+    this.trackLastFrameTime.forEach((lastTime) => {
+      if (now - lastTime < FRAME_INACTIVITY_TIMEOUT) {
+        hasActiveTrack = true;
+      }
+    });
+
+    if (!hasActiveTrack && this.isReceiving) {
+      console.log('[MediaStreamHandler] Frame inactivity detected, connection may be stale');
+      this.isReceiving = false;
+
+      // Trigger reconnect
+      if (!this.isClosed && this.pcManager) {
+        this.pcManager.reconnect();
+      }
+    }
+  }
+
+  private handlePeerConnected(): void {
+    this.isConnecting = false;
+    // Peer connection established - media setup will follow
+  }
+
+  private handlePeerDisconnected(): void {
+    this.isReceiving = false;
+    this.onDisconnectedCallback?.();
+  }
+
+  private handleReconnected(attempt: number): void {
+    // Tracks are re-added via onPeerConnectionCreated when new peer connection is created
+    // Just notify callback that reconnection succeeded
+    console.log(`[MediaStreamHandler] Reconnected after ${attempt} attempts`);
+    this.onReconnectedCallback?.(attempt);
+  }
+
+  private handleError(error: Error): void {
+    console.error('[MediaStreamHandler] MediaStream error:', error);
+    this.onErrorCallback?.(error);
+  }
+
+  private notifyTrackListeners(track: MediaStreamTrack): void {
+    this.trackListeners.forEach((listener) => {
+      try {
+        listener(track);
+      } catch (err) {
+        console.error('[MediaStreamHandler] Error in track listener:', err);
+      }
+    });
+  }
+
+  private notifyFrameListeners(frameData: any): void {
+    this.frameListeners.forEach((listener) => {
+      try {
+        listener(frameData);
+      } catch (err) {
+        console.error('[MediaStreamHandler] Error in frame listener:', err);
+      }
+    });
+  }
+
+  private notifyCloseListeners(): void {
+    this.closeListeners.forEach((listener) => {
+      try {
+        listener();
+      } catch (err) {
+        console.error('[MediaStreamHandler] Error in close listener:', err);
+      }
+    });
+  }
+
+  private createPhygridMediaStream(): PhygridMediaStream {
+    return {
+      getTracks: () => [...this.tracks],
+
+      getStream: () => this.remoteStream,
+
+      addTrack: (track: MediaStreamTrack) => {
+        if (this.isClosed) {
+          console.warn('[MediaStreamHandler] Cannot add track to closed stream');
+          return;
+        }
+
+        // Add to local stream
+        if (this.localStream) {
+          const existing = this.localStream.getTrackById(track.id);
+          if (existing) {
+            this.localStream.removeTrack(existing);
+          }
+          this.localStream.addTrack(track);
+        }
+
+        // Add to tracks collection
+        this.tracks.push(track);
+
+        // Add to peer connection if connected
+        const pc = this.pcManager?.getPeerConnection();
+        if (pc && pc.connectionState === 'connected' && this.localStream) {
+          const senders = pc.getSenders();
+          const existingSender = senders.find((sender) => sender.track?.id === track.id);
+          if (existingSender) {
+            existingSender.replaceTrack(track);
+          } else {
+            pc.addTrack(track, this.localStream);
+          }
+        }
+
+        this.notifyTrackListeners(track);
+      },
+
+      onTrack: (callback: (track: MediaStreamTrack) => void) => {
+        this.trackListeners.add(callback);
+
+        // Notify for existing tracks
+        this.tracks.forEach((track) => {
+          try {
+            callback(track);
+          } catch (err) {
+            console.error('[MediaStreamHandler] Error in track listener:', err);
+          }
+        });
+      },
+
+      offTrack: (callback: (track: MediaStreamTrack) => void) => {
+        this.trackListeners.delete(callback);
+      },
+
+      onFrame: (callback: (frameData: any) => void) => {
+        this.frameListeners.add(callback);
+      },
+
+      offFrame: (callback: (frameData: any) => void) => {
+        this.frameListeners.delete(callback);
+      },
+
+      onClose: (callback: () => void) => {
+        this.closeListeners.add(callback);
+      },
+
+      offClose: (callback: () => void) => {
+        this.closeListeners.delete(callback);
+      },
+
+      close: () => {
+        this.close();
+      },
+
+      isReceivingFrames: () => {
+        if (this.trackLastFrameTime.size === 0) return false;
+
+        const now = Date.now();
+        for (const lastTime of this.trackLastFrameTime.values()) {
+          if (now - lastTime < FRAME_INACTIVITY_TIMEOUT) {
+            return true;
+          }
+        }
+        return false;
+      },
+
+      getTargetTwinId: () => {
+        return this.targetTwinId;
+      },
+
+      getChannelName: () => {
+        return this.channelName;
+      },
+
+      isConnecting: () => {
+        return this.isConnecting;
+      },
+    };
+  }
+}