@dronedeploy/rocos-js-sdk 1.0.0-alpha-2 → 2.6.1

package/demo/angular/src/app/components/video/p2pvideo/index.ts

@@ -0,0 +1,646 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/explicit-member-accessibility */
+
+import { Subject } from 'rxjs'
+import {
+  WebRTCSignallingService,
+  IOperatorConnectRequest,
+  IStatus,
+  IWebRTCSignallingMessage,
+  IGetDetailsRequest,
+  IOfferRequest,
+  IServiceError,
+  IWebRTCSignallingOutcome,
+  IAddIceCandidateRequest,
+  IWebRTCSignallingResponseStream,
+} from '@team-rocos/rocos-js-sdk'
+import { SdkService } from '../../../services/sdk.service'
+import { grpc } from '@improbable-eng/grpc-web'
+import { parseStats, IStatsResult, IVideoSourceStatus, VideoSourceStatusEnum } from '../video-source'
+
+/**
+ * P2P video service connection service
+ */
+export class P2PVideoService {
+  // The detailed status for the video source
+  public videoSourceStatus: Subject<IVideoSourceStatus> = new Subject<IVideoSourceStatus>()
+
+  // Status to indicate the streaming and connecting
+  public isVideoStreaming: Subject<boolean> = new Subject<boolean>()
+  public isVideoConnecting: Subject<boolean> = new Subject<boolean>()
+
+  // The Obersable subject that used to stream the the actual WebRTC video stream once available
+  public mediaStream = new Subject<MediaStream | undefined>()
+
+  public connectingMessage?: string
+
+  //#region Private fields
+
+  private _webRTCSignallingService: WebRTCSignallingService
+
+  // The video operator stream that connects to rocos Pigeon gRPC service
+  private _videoOperatorStream?: IWebRTCSignallingResponseStream<IWebRTCSignallingMessage>
+
+  // Internal indicator for checking if the stop stream has been called
+  private _isStopped: boolean = false
+
+  // The RTC peer connection object provided by the browser to coordinate the WebRTC video stream
+  private _peerConnection?: RTCPeerConnection
+
+  // The detailed WebRTC stats that collected by application
+  private _webRTCStats?: IStatsResult
+  private _lastBitrateRecordTime?: number
+  private _bitrateKbps: number = 0
+
+  private _lastJitterBufferDelay: number = 0
+  private _lastJitterBufferEmittedCount: number = 0
+
+  private _videoSourceStatus: IVideoSourceStatus = {}
+
+  private _videoStreaming: boolean = false
+  private _videoConnecting: boolean = false
+  private _awaitAnswerTimeout?: number
+  private _awaitPeerConnectionTimeout?: number
+  private _retryWatchTimeout?: number
+  private _updateStatsTimerInterval?: number
+  private _retryTimerInterval?: number
+
+  constructor(
+    private sdkService: SdkService,
+    private projectId: string,
+    private videoId: string,
+    private videoCommand: string,
+    private playoutDelayHint: string,
+    private iceServers: any,
+  ) {
+    this._webRTCSignallingService = this.sdkService.getWebRTCSignallingService()
+  }
+
+  /**
+   * Function to start the video stream
+   * @returns
+   */
+  public startStream(): void {
+    if (!this.videoId || !this.projectId || !this.videoCommand) {
+      console.error('Required parameters are missing, cannot start video stream')
+      this.stopStream(false, 'Required parameters are missing')
+      return
+    }
+
+    // this.webRTCSignallingService.getDetails()
+    const cameraStatusRequest: IGetDetailsRequest = {
+      videoId: this.videoId,
+      projectId: this.projectId,
+    }
+
+    // Get the connections details to check if the current connection can be performed
+    this._webRTCSignallingService.getDetails(cameraStatusRequest, async (_, response) => {
+      const operators = response ? response.getOperators() : 0
+
+      // There is other operator already connected to the video, either force the other operator to disconnect or cancel current request
+      if (operators > 0) {
+        const title = 'Existing Connection Detected'
+
+        // eslint-disable-next-line max-len
+        const message = `There is someone already connected to this camera. If you continue to connect, they will be disconnected.`
+
+        const confirmResult = window.confirm(message)
+
+        if (confirmResult) {
+          await this._startWatching()
+        }
+      } else {
+        await this._startWatching()
+      }
+    })
+  }
+
+  /**
+   * Stop the vide stream
+   * @param restartHasScheduled
+   * @param msg
+   */
+  public stopStream(restartHasScheduled: boolean = false, msg?: string): void {
+    console.log('Stopping stream')
+
+    this._isStopped = true
+
+    if (this._videoOperatorStream) {
+      try {
+        this._videoOperatorStream.cancel()
+      } catch (ex) {
+        console.warn('Failed to cancel the operator stream, the stream might already cancelled', { ex })
+      }
+    }
+
+    this.connectingMessage = msg
+    this._stopInternal(restartHasScheduled)
+  }
+
+  /**
+   * Start establishing the required WebRTC signalling service and video stream
+   * @returns
+   */
+  private async _startWatching(): Promise<void> {
+    if (this._isStopped) {
+      return
+    }
+
+    this._clearInternalTimers()
+
+    this.updateVideoStreaming(false)
+    this.updateVideoConnecting(true)
+
+    this.connectingMessage = 'Initiating...'
+
+    const request: IOperatorConnectRequest = { videoId: this.videoId, projectId: this.projectId }
+
+    // connect as an operator (disconnects others)...
+    this._videoOperatorStream = await this._webRTCSignallingService.connectOperator(request)
+
+    // Once the operator connected, it will wait for the camera to be connected.
+    // The camera could already connected to Pigeon through agent or not online yet. (Agent automatcially connect the cameras to Pigeon service as soon as it's online)
+    console.log('Connected to Pigeon Operator Service. Waiting for camera...')
+    console.log(`Waiting for camera...`)
+
+    this.connectingMessage = 'Waiting for camera...'
+
+    this._videoOperatorStream.on('status', (status: IStatus) => {
+      if (status.code === grpc.Code.OK) {
+        console.log('Video operator stream status changed to ok')
+        return
+      }
+
+      if (this._awaitAnswerTimeout) {
+        clearTimeout(this._awaitAnswerTimeout)
+      }
+      if (this._awaitPeerConnectionTimeout) {
+        clearTimeout(this._awaitPeerConnectionTimeout)
+      }
+
+      console.log(`Signaling operator in error status`, { status })
+
+      if (status.code === grpc.Code.ResourceExhausted) {
+        // The disconnection might be interapted by another operator watchning.
+        // We only want to stop stream but not restarting watching again.
+        this._bootHandler()
+      } else {
+        this._restartWatching(`Netowrk error with code ${status.code}`)
+      }
+    })
+
+    // WebRTC signalling service sent data to the operator
+    this._videoOperatorStream.on('data', (message: IWebRTCSignallingMessage) => {
+      const messageType = message.getMessagetype()
+      console.log(`Received '${messageType}' message`)
+
+      switch (messageType) {
+        case 'camera-connected': {
+          console.log('Camera connected')
+          this.connectingMessage = 'Camera online. Waiting for video stream...'
+
+          this._setupPeerConnection()
+
+          // Setup peer connection requires a timout watcher to deal with error if the agent doesn't reply the initial offering request in time
+          this._awaitAnswerTimeout = window.setTimeout(
+            (msg: string) => {
+              this._restartWatching(msg)
+            },
+            10000,
+            'Timeout waiting for answer from signaling server',
+          )
+          break
+        }
+        // WebRTC signalling service replied the connection request after received the response from agent
+        case 'answer': {
+          clearTimeout(this._awaitAnswerTimeout)
+          const payload = message.getPayload()
+          const answer = JSON.parse(payload)
+
+          // Process the answer
+          this._answerHandler(answer)
+
+          // Watch if the agent failed to establish the video stream with the operation
+          this._awaitPeerConnectionTimeout = setTimeout(
+            (msg: string) => {
+              this._restartWatching(msg)
+            },
+            10000,
+            'Timeout waiting for peer connection',
+          )
+          break
+        }
+      }
+    })
+  }
+
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+  private _answerHandler(answerRequest: any): void {
+    if (this._isStopped) {
+      return
+    }
+
+    const remoteDescription = JSON.parse(atob(answerRequest.sdp))
+
+    if (this._peerConnection) {
+      this._peerConnection
+        .setRemoteDescription(new RTCSessionDescription(remoteDescription))
+        .then(() => {
+          // will not be called immediately if a connection is already underway
+          // log('Finished setRemoteDescription()');
+        })
+        .catch((err) => {
+          console.log(err)
+        })
+    }
+  }
+
+  /**
+   * Handle the event that the current connection has been disconnected by other operator
+   * @returns
+   */
+  private _bootHandler(): void {
+    if (this._isStopped) {
+      return
+    }
+    this.connectingMessage = 'Another operator has taken over the connection.'
+    this._stopInternal(false)
+  }
+
+  /**
+   * Restart watching if the video was disconnected due to temp error
+   * @param message
+   */
+  private _restartWatching(message: string): void {
+    console.log(`restart watching: projectId: ${this.projectId}`)
+    this.stopStream(true, message + ': Restarting connection...')
+
+    this._retryWatchTimeout = window.setTimeout(() => this._startWatching(), 3000)
+  }
+
+  /**
+   * Set the peer connection between operator on agent
+   */
+  private _setupPeerConnection(): void {
+    this._peerConnection = new RTCPeerConnection({
+      iceServers: this.iceServers,
+    })
+
+    // Video stream track is available
+    this._peerConnection.ontrack = (ev: RTCTrackEvent) => {
+      // eslint-disable-next-line max-len
+      console.log(
+        `PeerConnection received a streams containing ${ev.streams[0].getAudioTracks().length} audio tracks(s) and ${
+          ev.streams[0].getVideoTracks().length
+        } video tracks(s)`,
+      )
+
+      // Forward the media stream to the consumer
+      this.mediaStream.next(ev.streams[0])
+
+      if (this.playoutDelayHint) {
+        if (/^\d+$/.test(this.playoutDelayHint)) {
+          // playoutDelayHint value is in second, convert from mili second to second
+          const playoutDelayHint = parseInt(this.playoutDelayHint, 10) / 1000
+          if (playoutDelayHint !== 0) {
+            const receivers = (this._peerConnection as any).getReceivers()
+            // Set the playoutDelayHint
+            if (receivers.length > 0 && !receivers[0].playoutDelayHint) {
+              receivers[0].playoutDelayHint = playoutDelayHint
+            }
+          }
+        }
+      }
+    }
+
+    // Debugging function to track the state change event
+    this._peerConnection.onsignalingstatechange = () => {
+      if (this._peerConnection) {
+        console.log(`PeerConnection signaling state changed to '${this._peerConnection.signalingState}'`)
+      }
+    }
+
+    // ICE connection state tracking function
+    this._peerConnection.oniceconnectionstatechange = () => {
+      if (this._peerConnection) {
+        console.log(`PeerConnection ice connection state changed to '${this._peerConnection.iceConnectionState}'`)
+      }
+
+      if (
+        this._peerConnection &&
+        (this._peerConnection.iceConnectionState === 'failed' ||
+          this._peerConnection.iceConnectionState === 'disconnected' ||
+          this._peerConnection.iceConnectionState === 'closed')
+      ) {
+        console.log('****************************************************')
+        console.log('the peer connection failed and we need to handle it')
+        console.log('****************************************************')
+
+        // Print out error log, the acutally error state handler is at a different event track function
+      }
+
+      if (this._peerConnection && this._peerConnection.iceConnectionState === 'connected') {
+        // Clear the time out watcher once the state become connected
+        if (this._awaitAnswerTimeout) {
+          clearTimeout(this._awaitAnswerTimeout)
+        }
+        if (this._awaitPeerConnectionTimeout) {
+          clearTimeout(this._awaitPeerConnectionTimeout)
+        }
+      }
+    }
+
+    // ICE gathering state tracking function
+    this._peerConnection.onicegatheringstatechange = () => {
+      if (this._peerConnection) {
+        console.log(`PeerConnection ice gathering state changed to '${this._peerConnection.iceGatheringState}'`)
+      }
+    }
+
+    // ICE candidate error tracking function
+    this._peerConnection.onicecandidateerror = (event: any) => {
+      console.log(`PeerConnection error occurred during ICE candidate gathering: Code: '${event.errorCode}'`)
+    }
+
+    // Create the actual camera connection offer once the peer connection requiring negotiation
+    this._peerConnection.onnegotiationneeded = () => {
+      this._createAndSendOffer(null)
+    }
+
+    // When ICE Candidate alailable, sende the ICE Candidates to agent through signalling service
+    this._peerConnection.onicecandidate = (event) => {
+      if (event.candidate && event.candidate.candidate) {
+        const request: IAddIceCandidateRequest = {
+          videoId: this.videoId,
+          candidate: event.candidate.candidate,
+          projectId: this.projectId,
+        }
+
+        // Send the ICE candidate to agent through signalling service
+        this._webRTCSignallingService.addIceCandidate(request, (err, response) => {
+          if (err) {
+            console.log('this.operatorService.addIceCandidate ERROR:')
+            console.log(err)
+          }
+        })
+      } else {
+        // All candidates received
+        console.log('All candidates received!')
+      }
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    // Track the connection stats change events and apply logic accordingly
+    this._peerConnection.onconnectionstatechange = (_) => {
+      // "new" | "connecting" | "connected" | "disconnected" | "failed" | "closed";
+
+      const connectionState = this._peerConnection?.connectionState
+
+      console.log(`PeerConnection connection state changed to '${connectionState}'`)
+
+      switch (connectionState) {
+        case 'failed':
+        case 'disconnected':
+          this.connectingMessage = `${connectionState}: retrying...`
+
+          this._restartWatching(`Stream ${connectionState}`)
+          break
+        case 'connected':
+          this.connectingMessage = undefined
+          this.updateVideoStreaming(true)
+          this.updateVideoConnecting(false)
+          break
+        default:
+          this.connectingMessage = connectionState
+      }
+    }
+
+    this._peerConnection.addTransceiver('video', { 'direction': 'recvonly' })
+
+    if (this._updateStatsTimerInterval) {
+      clearInterval(this._updateStatsTimerInterval)
+    }
+
+    // Start updating the state every 1 sec
+    this._updateStatsTimerInterval = window.setInterval(() => this._updateStats(), 1000)
+  }
+
+  /**
+   * Create and send the inital connection offer
+   * @param offerOptions
+   * @returns
+   */
+  private async _createAndSendOffer(offerOptions: RTCOfferOptions | null): Promise<void> {
+    offerOptions = offerOptions || { iceRestart: false }
+
+    if (!this._peerConnection) {
+      console.error('Peer connection cannot be null when creating the offer')
+      this._restartWatching('Peer connection cannot be null when creating the offer')
+      return
+    }
+
+    // Create and send offer based on the PeerConnection info
+    try {
+      const offer = await this._peerConnection.createOffer(offerOptions)
+
+      await this._peerConnection.setLocalDescription(offer)
+
+      this._sendOffer(this._peerConnection.localDescription as RTCSessionDescription)
+    } catch (err) {
+      console.error('Failed to create or send offer', { err })
+    }
+  }
+
+  /**
+   * Send offer to agent through WebRTC signalling service
+   * @param offer
+   * @returns
+   */
+  private _sendOffer(offer?: RTCSessionDescription): void {
+    if (offer == null) {
+      this._restartWatching('Connection off cannot be null')
+      return
+    }
+
+    const request: IOfferRequest = {
+      projectId: this.projectId as string,
+      rtcSessionDescription: offer,
+      videoCommand: this.videoCommand,
+      videoId: this.videoId,
+      clientId: '',
+      videoFormat: 'H264',
+    }
+
+    console.log('Offer request', { request })
+
+    this._webRTCSignallingService.offerConnection(
+      request,
+      (err: IServiceError | null, response: IWebRTCSignallingOutcome | null) => {
+        if (err) {
+          console.error('Send offer connection failed', { err })
+        }
+
+        if (response) {
+          console.log('Send offer connection succeeded', { response })
+        }
+      },
+    )
+  }
+
+  /**
+   * Update stats with the info from peer connection
+   */
+  private async _updateStats(): Promise<void> {
+    let diff = 1000
+
+    if (this._lastBitrateRecordTime) {
+      const now = window.performance.now()
+      diff = now - this._lastBitrateRecordTime
+    }
+
+    this._videoSourceStatus.realInterval = diff
+
+    this._lastBitrateRecordTime = window.performance.now()
+    if (this._peerConnection) {
+      const stats = await this._peerConnection.getStats(null)
+
+      const statsResult: IStatsResult = parseStats(stats)
+
+      this._videoSourceStatus.webrtcRoundTripTimeMs = statsResult.bandwidth.currentRoundTripTime * 1000
+
+      this._videoSourceStatus.webrtcPeerConnectionStatus = this._peerConnection.connectionState === 'connected'
+      this._videoSourceStatus.webrtcIceStateConnected =
+        ['connected', 'completed'].indexOf(this._peerConnection.iceConnectionState) > 0
+      this._videoSourceStatus.webrtcIceConnectionState =
+        this._peerConnection.iceConnectionState[0].toUpperCase() + this._peerConnection.iceConnectionState.substr(1)
+
+      if (statsResult.video.packetsReceived > 0) {
+        this._videoSourceStatus.webrtcPacketLossPercent = +(
+          (statsResult.video.packetsLost / statsResult.video.packetsReceived) *
+          100
+        ).toFixed(2)
+      }
+
+      if (this._peerConnection.getTransceivers().length > 0) {
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        const transceiver = (<any>this._peerConnection).getTransceivers()[0]
+        this._videoSourceStatus.webrtcStreamStarted = !transceiver.stopped
+      } else {
+        this._videoSourceStatus.webrtcStreamStarted = false
+      }
+
+      this._videoSourceStatus.webrtcVideoIn = statsResult.video.recv.tracks.length > 0
+
+      if (this._webRTCStats && statsResult.video.bytesReceived && this._webRTCStats.video.bytesReceived) {
+        if (this._webRTCStats.video.bytesReceived > statsResult.video.bytesReceived) {
+          this._webRTCStats.video.bytesReceived = statsResult.video.bytesReceived
+        }
+        // multiply to turbo because we may changed the update rate
+        const extraBytesReceived = statsResult.video.bytesReceived - this._webRTCStats.video.bytesReceived
+        const bytesPerSecond = (extraBytesReceived / this._videoSourceStatus.realInterval) * 1000
+        const kbitsPerSecond = bytesPerSecond / 128
+        this._bitrateKbps = kbitsPerSecond
+        this._videoSourceStatus.webrtcBitrate = `${Math.floor(this._bitrateKbps)}`
+      } else {
+        this._videoSourceStatus.webrtcBitrate = '0'
+      }
+
+      if (statsResult.video && statsResult.video.recv) {
+        if (statsResult.video.recv.framesDropped || statsResult.video.recv.framesDropped === 0) {
+          this._videoSourceStatus.webrtcFramesDropped = statsResult.video.recv.framesDropped
+        }
+        if (statsResult.video.recv.framesDecoded || statsResult.video.recv.framesDecoded === 0) {
+          this._videoSourceStatus.webrtcFramesDecoded = statsResult.video.recv.framesDecoded
+        }
+        if (statsResult.video.recv.framesReceived || statsResult.video.recv.framesReceived === 0) {
+          this._videoSourceStatus.webrtcFramesReceived = statsResult.video.recv.framesReceived
+        }
+        if (statsResult.video.recv.jitterBufferDelay) {
+          this._videoSourceStatus.jitterBufferDelay = statsResult.video.recv.jitterBufferDelay
+        }
+        if (statsResult.video.recv.jitterBufferEmittedCount) {
+          this._videoSourceStatus.jitterBufferEmittedCount = statsResult.video.recv.jitterBufferEmittedCount
+        }
+
+        if (statsResult.video.recv.jitterBufferDelay && statsResult.video.recv.jitterBufferEmittedCount) {
+          this._videoSourceStatus.jitterBufferEmittedCount = statsResult.video.recv.jitterBufferEmittedCount
+
+          const currentJitterBufferDelay = statsResult.video.recv.jitterBufferDelay
+          const delayThisPeriod = currentJitterBufferDelay - this._lastJitterBufferDelay
+          this._lastJitterBufferDelay = currentJitterBufferDelay
+
+          const currentJitterBufferEmittedCount = statsResult.video.recv.jitterBufferEmittedCount
+          const countThisPeriod = currentJitterBufferEmittedCount - this._lastJitterBufferEmittedCount
+          this._lastJitterBufferEmittedCount = currentJitterBufferEmittedCount
+
+          const avgDelayThisPeriod = delayThisPeriod / countThisPeriod
+          this._videoSourceStatus.avgFramePlayout = avgDelayThisPeriod
+        } else {
+          this._videoSourceStatus.avgFramePlayout = undefined
+        }
+      }
+
+      this._webRTCStats = statsResult
+      this.updateVideoSourceStatus()
+    }
+  }
+
+  // Clear all timers
+  private _clearInternalTimers(): void {
+    if (this._awaitAnswerTimeout) {
+      clearTimeout(this._awaitAnswerTimeout)
+    }
+    if (this._awaitPeerConnectionTimeout) {
+      clearTimeout(this._awaitPeerConnectionTimeout)
+    }
+    if (this._updateStatsTimerInterval) {
+      clearInterval(this._updateStatsTimerInterval)
+    }
+    if (this._retryWatchTimeout) {
+      clearTimeout(this._retryWatchTimeout)
+    }
+    if (this._retryTimerInterval) {
+      clearInterval(this._retryTimerInterval)
+    }
+  }
+
+  private _stopInternal(restartHasScheduled?: boolean): void {
+    // this is called when booted
+    // don't send stop command to server - it will disconnect the other viewer
+    this.updateVideoStreaming(false)
+    this.updateVideoConnecting(restartHasScheduled)
+
+    this.mediaStream.next(undefined)
+
+    if (this._peerConnection) {
+      this._peerConnection.close()
+      this._peerConnection = undefined
+    }
+
+    this._clearInternalTimers()
+  }
+
+  private updateVideoSourceStatus(key?: VideoSourceStatusEnum, value?: any) {
+    if (key !== undefined && value !== undefined) {
+      this._videoSourceStatus[key] = value
+    }
+
+    this.videoSourceStatus.next(this._videoSourceStatus)
+  }
+
+  private updateVideoStreaming(streaming?: boolean) {
+    if (streaming !== undefined) {
+      this._videoStreaming = streaming
+    }
+
+    this.isVideoStreaming.next(this._videoStreaming)
+  }
+
+  private updateVideoConnecting(connecting?: boolean) {
+    if (connecting !== undefined) {
+      this._videoConnecting = connecting
+    }
+
+    this.isVideoConnecting.next(this._videoConnecting)
+  }
+}