@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.1.3 → 0.2.1

package/src/PeerConnectionController/PeerConnectionController.ts

@@ -23,11 +23,7 @@ export class PeerConnectionController {
      * @param options - Peer connection Options
      * @param config - The config for our PS experience.
      */
-    constructor(
-        options: RTCConfiguration,
-        config: Config,
-        preferredCodec: string
-    ) {
+    constructor(options: RTCConfiguration, config: Config, preferredCodec: string) {
         this.config = config;
         this.createPeerConnection(options, preferredCodec);
     }
@@ -36,26 +32,18 @@ export class PeerConnectionController {
         // Set the ICE transport to relay if TURN enabled
         if (this.config.isFlagEnabled(Flags.ForceTURN)) {
             options.iceTransportPolicy = 'relay';
-            Logger.Log(
-                Logger.GetStackTrace(),
-                'Forcing TURN usage by setting ICE Transport Policy in peer connection config.'
-            );
+            Logger.Info('Forcing TURN usage by setting ICE Transport Policy in peer connection config.');
         }
 
         // build a new peer connection with the options
         this.peerConnection = new RTCPeerConnection(options);
-        this.peerConnection.onsignalingstatechange = (ev: Event) =>
-            this.handleSignalStateChange(ev);
+        this.peerConnection.onsignalingstatechange = (ev: Event) => this.handleSignalStateChange(ev);
         this.peerConnection.oniceconnectionstatechange = (ev: Event) =>
             this.handleIceConnectionStateChange(ev);
-        this.peerConnection.onicegatheringstatechange = (ev: Event) =>
-            this.handleIceGatheringStateChange(ev);
-        this.peerConnection.ontrack = (ev: RTCTrackEvent) =>
-            this.handleOnTrack(ev);
-        this.peerConnection.onicecandidate = (ev: RTCPeerConnectionIceEvent) =>
-            this.handleIceCandidate(ev);
-        this.peerConnection.ondatachannel = (ev: RTCDataChannelEvent) =>
-            this.handleDataChannel(ev);
+        this.peerConnection.onicegatheringstatechange = (ev: Event) => this.handleIceGatheringStateChange(ev);
+        this.peerConnection.ontrack = (ev: RTCTrackEvent) => this.handleOnTrack(ev);
+        this.peerConnection.onicecandidate = (ev: RTCPeerConnectionIceEvent) => this.handleIceCandidate(ev);
+        this.peerConnection.ondatachannel = (ev: RTCDataChannelEvent) => this.handleDataChannel(ev);
         this.aggregatedStats = new AggregatedStats();
         this.preferredCodec = preferredCodec;
         this.updateCodecSelection = true;
@@ -66,26 +54,24 @@ export class PeerConnectionController {
      * @param offerOptions - RTC Offer Options
      */
     async createOffer(offerOptions: RTCOfferOptions, config: Config) {
-        Logger.Log(Logger.GetStackTrace(), 'Create Offer', 6);
+        Logger.Info('Create Offer');
 
-        const isLocalhostConnection =
-            location.hostname === 'localhost' ||
-            location.hostname === '127.0.0.1';
+        const isLocalhostConnection = location.hostname === 'localhost' || location.hostname === '127.0.0.1';
         const isHttpsConnection = location.protocol === 'https:';
         let useMic = config.isFlagEnabled(Flags.UseMic);
-        if (useMic && !(isLocalhostConnection || isHttpsConnection)) {
+        let useCamera = config.isFlagEnabled(Flags.UseCamera);
+        if ((useMic || useCamera) && !(isLocalhostConnection || isHttpsConnection)) {
             useMic = false;
+            useCamera = false;
             Logger.Error(
-                Logger.GetStackTrace(),
-                'Microphone access in the browser will not work if you are not on HTTPS or localhost. Disabling mic access.'
+                'Microphone and Webcam access in the browser will not work if you are not on HTTPS or localhost. Disabling mic and webcam access.'
             );
             Logger.Error(
-                Logger.GetStackTrace(),
                 "For testing you can enable HTTP microphone access Chrome by visiting chrome://flags/ and enabling 'unsafely-treat-insecure-origin-as-secure'"
             );
         }
 
-        this.setupTransceiversAsync(useMic).finally(() => {
+        this.setupTransceiversAsync(useMic, useCamera).finally(() => {
             this.peerConnection
                 ?.createOffer(offerOptions)
                 .then((offer: RTCSessionDescriptionInit) => {
@@ -104,27 +90,32 @@ export class PeerConnectionController {
      *
      */
     async receiveOffer(offer: RTCSessionDescriptionInit, config: Config) {
-        Logger.Log(Logger.GetStackTrace(), 'Receive Offer', 6);
+        Logger.Info('Receive Offer');
 
         this.peerConnection?.setRemoteDescription(offer).then(() => {
             const isLocalhostConnection =
-                location.hostname === 'localhost' ||
-                location.hostname === '127.0.0.1';
+                location.hostname === 'localhost' || location.hostname === '127.0.0.1';
             const isHttpsConnection = location.protocol === 'https:';
             let useMic = config.isFlagEnabled(Flags.UseMic);
-            if (useMic && !(isLocalhostConnection || isHttpsConnection)) {
+            let useCamera = config.isFlagEnabled(Flags.UseCamera);
+            if ((useMic || useCamera) && !(isLocalhostConnection || isHttpsConnection)) {
                 useMic = false;
+                useCamera = false;
                 Logger.Error(
-                    Logger.GetStackTrace(),
-                    'Microphone access in the browser will not work if you are not on HTTPS or localhost. Disabling mic access.'
+                    'Microphone and Webcam access in the browser will not work if you are not on HTTPS or localhost. Disabling mic and webcam access.'
                 );
                 Logger.Error(
-                    Logger.GetStackTrace(),
                     "For testing you can enable HTTP microphone access Chrome by visiting chrome://flags/ and enabling 'unsafely-treat-insecure-origin-as-secure'"
                 );
             }
 
-            this.setupTransceiversAsync(useMic).finally(() => {
+            // Add our list of preferred codecs, in order of preference
+            this.config.setOptionSettingOptions(
+                OptionParameters.PreferredCodec,
+                this.fuzzyIntersectUEAndBrowserCodecs(offer)
+            );
+
+            this.setupTransceiversAsync(useMic, useCamera).finally(() => {
                 this.peerConnection
                     ?.createAnswer()
                     .then((Answer: RTCSessionDescriptionInit) => {
@@ -132,28 +123,13 @@ export class PeerConnectionController {
                         return this.peerConnection?.setLocalDescription(Answer);
                     })
                     .then(() => {
-                        this.onSendWebRTCAnswer(
-                            this.peerConnection?.currentLocalDescription
-                        );
+                        this.onSendWebRTCAnswer(this.peerConnection?.currentLocalDescription);
                     })
                     .catch(() => {
-                        Logger.Error(
-                            Logger.GetStackTrace(),
-                            'createAnswer() failed'
-                        );
+                        Logger.Error('createAnswer() failed');
                     });
             });
         });
-
-        // Ugly syntax, but this achieves the intersection of the browser supported list and the UE supported list
-        this.config.setOptionSettingOptions(
-            OptionParameters.PreferredCodec,
-            this.parseAvailableCodecs(offer).filter((value) =>
-                this.config
-                    .getSettingOption(OptionParameters.PreferredCodec)
-                    .options.includes(value)
-            )
-        );
     }
 
     /**
@@ -162,14 +138,11 @@ export class PeerConnectionController {
      */
     receiveAnswer(answer: RTCSessionDescriptionInit) {
         this.peerConnection?.setRemoteDescription(answer);
-        // Ugly syntax, but this achieves the intersection of the browser supported list and the UE supported list
+
+        // Add our list of preferred codecs, in order of preference
         this.config.setOptionSettingOptions(
             OptionParameters.PreferredCodec,
-            this.parseAvailableCodecs(answer).filter((value) =>
-                this.config
-                    .getSettingOption(OptionParameters.PreferredCodec)
-                    .options.includes(value)
-            )
+            this.fuzzyIntersectUEAndBrowserCodecs(answer)
         );
     }
 
@@ -190,9 +163,7 @@ export class PeerConnectionController {
             if (this.updateCodecSelection && !!this.aggregatedStats.inboundVideoStats.codecId) {
                 this.config.setOptionSettingValue(
                     OptionParameters.PreferredCodec,
-                    this.aggregatedStats.codecs.get(
-                        this.aggregatedStats.inboundVideoStats.codecId
-                    )
+                    this.aggregatedStats.codecs.get(this.aggregatedStats.inboundVideoStats.codecId)
                 );
             }
         });
@@ -229,9 +200,7 @@ export class PeerConnectionController {
         }
 
         // Force mono or stereo based on whether ?forceMono was passed or not
-        audioSDP += this.config.isFlagEnabled(Flags.ForceMonoAudio)
-            ? 'stereo=0;'
-            : 'stereo=1;';
+        audioSDP += this.config.isFlagEnabled(Flags.ForceMonoAudio) ? 'stereo=0;' : 'stereo=1;';
 
         // enable in-band forward error correction for opus audio
         audioSDP += 'useinbandfec=1';
@@ -247,16 +216,14 @@ export class PeerConnectionController {
      * @param iceCandidate - RTC Ice Candidate from the Signaling Server
      */
     handleOnIce(iceCandidate: RTCIceCandidate) {
-        Logger.Log(Logger.GetStackTrace(), 'peerconnection handleOnIce', 6);
+        Logger.Info('peerconnection handleOnIce');
 
         // // if forcing TURN, reject any candidates not relay
         if (this.config.isFlagEnabled(Flags.ForceTURN)) {
             // check if no relay address is found, if so, we are assuming it means no TURN server
             if (iceCandidate.candidate.indexOf('relay') < 0) {
                 Logger.Info(
-                    Logger.GetStackTrace(),
-                    `Dropping candidate because it was not TURN relay. | Type= ${iceCandidate.type} | Protocol= ${iceCandidate.protocol} | Address=${iceCandidate.address} | Port=${iceCandidate.port} |`,
-                    6
+                    `Dropping candidate because it was not TURN relay. | Type= ${iceCandidate.type} | Protocol= ${iceCandidate.protocol} | Address=${iceCandidate.address} | Port=${iceCandidate.port} |`
                 );
                 return;
             }
@@ -270,11 +237,7 @@ export class PeerConnectionController {
      * @param state - Signaling Server State Change Event
      */
     handleSignalStateChange(state: Event) {
-        Logger.Log(
-            Logger.GetStackTrace(),
-            'signaling state change: ' + state,
-            6
-        );
+        Logger.Info('signaling state change: ' + state);
     }
 
     /**
@@ -282,11 +245,7 @@ export class PeerConnectionController {
      * @param state - Ice Connection State
      */
     handleIceConnectionStateChange(state: Event) {
-        Logger.Log(
-            Logger.GetStackTrace(),
-            'ice connection state change: ' + state,
-            6
-        );
+        Logger.Info('ice connection state change: ' + state);
         this.onIceConnectionStateChange(state);
     }
 
@@ -295,11 +254,7 @@ export class PeerConnectionController {
      * @param state - Ice Gathering State Change
      */
     handleIceGatheringStateChange(state: Event) {
-        Logger.Log(
-            Logger.GetStackTrace(),
-            'ice gathering state change: ' + JSON.stringify(state),
-            6
-        );
+        Logger.Info('ice gathering state change: ' + JSON.stringify(state));
     }
 
     /**
@@ -371,18 +326,68 @@ export class PeerConnectionController {
         // Default Functionality: Do Nothing
     }
 
+    /**
+     * Find the intersection between UE and browser codecs, with fuzzy matching if some parameters are mismatched.
+     * @param sdp The remote sdp
+     * @returns The intersection between browser supported codecs and ue supported codecs.
+     */
+    fuzzyIntersectUEAndBrowserCodecs(sdp: RTCSessionDescriptionInit): string[] {
+        // We want to build an array of all supported codecs on both sides
+        const allSupportedCodecs: Array<string> = new Array<string>();
+        const allUECodecs: string[] = this.parseAvailableCodecs(sdp);
+        const allBrowserCodecs: string[] = this.config.getSettingOption(
+            OptionParameters.PreferredCodec
+        ).options;
+        for (const ueCodec of allUECodecs) {
+            // Check if browser codecs directly matches UE codec (with parameters and everything)
+            if (allBrowserCodecs.includes(ueCodec)) {
+                allSupportedCodecs.push(ueCodec);
+                continue;
+            }
+            // Otherwise check if browser codec at least contains a match for the UE codec name (without parameters).
+            else {
+                const ueCodecNameAndParams: string[] = ueCodec.split(' ');
+                const ueCodecName = ueCodecNameAndParams[0];
+                for (const browserCodec of allBrowserCodecs) {
+                    if (browserCodec.includes(ueCodecName)) {
+                        // We pass browser codec here as they option contain extra parameters.
+                        allSupportedCodecs.push(browserCodec);
+                        break;
+                    }
+                }
+            }
+        }
+        return allSupportedCodecs;
+    }
+
     /**
      * Setup tracks on the RTC Peer Connection
      * @param useMic - is mic in use
+     * @param useCamera - is webcam in use
      */
-    async setupTransceiversAsync(useMic: boolean) {
-        const hasTransceivers =
-            this.peerConnection?.getTransceivers().length > 0;
+    async setupTransceiversAsync(useMic: boolean, useCamera: boolean) {
+        let hasVideoReceiver = false;
+        for (const transceiver of this.peerConnection?.getTransceivers() ?? []) {
+            if (
+                transceiver &&
+                transceiver.receiver &&
+                transceiver.receiver.track &&
+                transceiver.receiver.track.kind === 'video'
+            ) {
+                hasVideoReceiver = true;
+                break;
+            }
+        }
 
-        // Setup a transceiver for getting UE video
-        this.peerConnection?.addTransceiver('video', { direction: 'recvonly' });
+        // Setup a transceiver for sending webcam video to UE and receiving video from UE
+        if (!useCamera) {
+            if (!hasVideoReceiver) {
+                this.peerConnection?.addTransceiver('video', { direction: 'recvonly' });
+            }
+        } else {
+            await this.setupVideoSender(hasVideoReceiver);
+        }
 
-        // We can only set preferred codec on Chrome
         if (RTCRtpReceiver.getCapabilities && this.preferredCodec != '') {
             for (const transceiver of this.peerConnection?.getTransceivers() ?? []) {
                 if (
@@ -390,101 +395,152 @@ export class PeerConnectionController {
                     transceiver.receiver &&
                     transceiver.receiver.track &&
                     transceiver.receiver.track.kind === 'video' &&
-                    // As of 06/2023, FireFox has added RTCRtpReceiver.getCapabilities, but hasn't added the ability to set codec preferences
                     transceiver.setCodecPreferences
                 ) {
+                    // Get our preferred codec from the codecs options drop down
                     const preferredRTPCodec = this.preferredCodec.split(' ');
-                    const codecs = [
-                        {
-                            mimeType:
-                                'video/' + preferredRTPCodec[0] /* Name */,
-                            clockRate: 90000,
-                            sdpFmtpLine: preferredRTPCodec[1] /* sdpFmtpLine */
-                                ? preferredRTPCodec[1]
-                                : ''
+                    const preferredRTCRtpCodecCapability: RTCRtpCodecCapability = {
+                        mimeType: 'video/' + preferredRTPCodec[0] /* Name */,
+                        clockRate: 90000 /* All current video formats in browsers have 90khz clock rate */,
+                        sdpFmtpLine: preferredRTPCodec[1] ? preferredRTPCodec[1] : ''
+                    };
+
+                    // Populate a list of codecs we will support with our preferred one in the first position
+                    const ourSupportedCodecs: Array<RTCRtpCodecCapability> = [preferredRTCRtpCodecCapability];
+
+                    // Go through all codecs the browser supports and add them to the list (in any order)
+                    RTCRtpReceiver.getCapabilities('video').codecs.forEach(
+                        (browserSupportedCodec: RTCRtpCodecCapability) => {
+                            // Don't add our preferred codec again, but add everything else
+                            if (browserSupportedCodec.mimeType != preferredRTCRtpCodecCapability.mimeType) {
+                                ourSupportedCodecs.push(browserSupportedCodec);
+                            } else if (
+                                browserSupportedCodec?.sdpFmtpLine !=
+                                preferredRTCRtpCodecCapability?.sdpFmtpLine
+                            ) {
+                                ourSupportedCodecs.push(browserSupportedCodec);
+                            }
                         }
-                    ];
-
-                    this.config
-                        .getSettingOption(OptionParameters.PreferredCodec)
-                        .options.filter((option) => {
-                            // Remove the preferred codec from the list of possible codecs as we've set it already
-                            return option != this.preferredCodec;
-                        })
-                        .forEach((option) => {
-                            // Amend the rest of the browsers supported codecs
-                            const altCodec = option.split(' ');
-                            codecs.push({
-                                mimeType: 'video/' + altCodec[0] /* Name */,
-                                clockRate: 90000,
-                                sdpFmtpLine: altCodec[1] /* sdpFmtpLine */
-                                    ? altCodec[1]
-                                    : ''
-                            });
-                        });
+                    );
 
-                    for (const codec of codecs) {
-                        if (codec.sdpFmtpLine === '') {
+                    for (const codec of ourSupportedCodecs) {
+                        if (codec?.sdpFmtpLine === undefined || codec.sdpFmtpLine === '') {
                             // We can't dynamically add members to the codec, so instead remove the field if it's empty
                             delete codec.sdpFmtpLine;
                         }
                     }
 
-                    transceiver.setCodecPreferences(codecs);
+                    transceiver.setCodecPreferences(ourSupportedCodecs);
                 }
             }
         }
 
+        let hasAudioReceiver = false;
+        for (const transceiver of this.peerConnection?.getTransceivers() ?? []) {
+            if (
+                transceiver &&
+                transceiver.receiver &&
+                transceiver.receiver.track &&
+                transceiver.receiver.track.kind === 'audio'
+            ) {
+                hasAudioReceiver = true;
+                break;
+            }
+        }
+
         // Setup a transceiver for sending mic audio to UE and receiving audio from UE
         if (!useMic) {
-            this.peerConnection?.addTransceiver('audio', {
-                direction: 'recvonly'
-            });
-        } else {
-            // set the audio options based on mic usage
-            const audioOptions = {
-                autoGainControl: false,
-                channelCount: 1,
-                echoCancellation: false,
-                latency: 0,
-                noiseSuppression: false,
-                sampleRate: 48000,
-                sampleSize: 16,
-                volume: 1.0
+            if (!hasAudioReceiver) {
+                this.peerConnection?.addTransceiver('audio', {
+                    direction: 'recvonly'
+                });
             }
+        } else {
+            await this.setupAudioSender(hasAudioReceiver);
+        }
+    }
 
-            // set the media send options
-            const mediaSendOptions: MediaStreamConstraints = {
-                video: false,
-                audio: audioOptions
-            };
+    async setupVideoSender(hasVideoReceiver: boolean) {
+        // set the media send options
+        const mediaSendOptions: MediaStreamConstraints = {
+            video: true
+        };
 
-            // Note using mic on android chrome requires SSL or chrome://flags/ "unsafely-treat-insecure-origin-as-secure"
-            const stream = await navigator.mediaDevices.getUserMedia(
-                mediaSendOptions
-            );
-            if (stream) {
-                if (hasTransceivers) {
-                    for (const transceiver of this.peerConnection?.getTransceivers() ?? []) {
-                        if (RTCUtils.canTransceiverReceiveAudio(transceiver)) {
-                            for (const track of stream.getTracks()) {
-                                if (track.kind && track.kind == 'audio') {
-                                    transceiver.sender.replaceTrack(track);
-                                    transceiver.direction = 'sendrecv';
-                                }
+        // Note using webcam on android chrome requires SSL or chrome://flags/ "unsafely-treat-insecure-origin-as-secure"
+        const stream = await navigator.mediaDevices.getUserMedia(mediaSendOptions);
+
+        if (stream) {
+            if (hasVideoReceiver) {
+                for (const transceiver of this.peerConnection?.getTransceivers() ?? []) {
+                    if (RTCUtils.canTransceiverReceiveVideo(transceiver)) {
+                        for (const track of stream.getTracks()) {
+                            if (track.kind && track.kind == 'video') {
+                                transceiver.sender.replaceTrack(track);
+                                transceiver.direction = 'sendrecv';
                             }
                         }
                     }
-                } else {
-                    for (const track of stream.getTracks()) {
-                        if (track.kind && track.kind == 'audio') {
-                            this.peerConnection?.addTransceiver(track, {
-                                direction: 'sendrecv'
-                            });
+                }
+            } else {
+                for (const track of stream.getTracks()) {
+                    if (track.kind && track.kind == 'video') {
+                        this.peerConnection?.addTransceiver(track, {
+                            direction: 'sendrecv'
+                        });
+                    }
+                }
+            }
+        } else {
+            if (!hasVideoReceiver) {
+                this.peerConnection?.addTransceiver('video', { direction: 'recvonly' });
+            }
+        }
+    }
+
+    async setupAudioSender(hasAudioReceiver: boolean) {
+        // set the audio options based on mic usage
+        const audioOptions = {
+            autoGainControl: false,
+            channelCount: 1,
+            echoCancellation: false,
+            latency: 0,
+            noiseSuppression: false,
+            sampleRate: 48000,
+            sampleSize: 16,
+            volume: 1.0
+        };
+
+        // set the media send options
+        const mediaSendOptions: MediaStreamConstraints = {
+            video: false,
+            audio: audioOptions
+        };
+
+        // Note using mic on android chrome requires SSL or chrome://flags/ "unsafely-treat-insecure-origin-as-secure"
+        const stream = await navigator.mediaDevices.getUserMedia(mediaSendOptions);
+        if (stream) {
+            if (hasAudioReceiver) {
+                for (const transceiver of this.peerConnection?.getTransceivers() ?? []) {
+                    if (RTCUtils.canTransceiverReceiveAudio(transceiver)) {
+                        for (const track of stream.getTracks()) {
+                            if (track.kind && track.kind == 'audio') {
+                                transceiver.sender.replaceTrack(track);
+                                transceiver.direction = 'sendrecv';
+                            }
                         }
                     }
                 }
             } else {
+                for (const track of stream.getTracks()) {
+                    if (track.kind && track.kind == 'audio') {
+                        this.peerConnection?.addTransceiver(track, {
+                            direction: 'sendrecv'
+                        });
+                    }
+                }
+            }
+        } else {
+            if (!hasAudioReceiver) {
                 this.peerConnection?.addTransceiver('audio', {
                     direction: 'recvonly'
                 });
@@ -533,12 +589,9 @@ export class PeerConnectionController {
         // Default Functionality: Do Nothing
     }
 
-    parseAvailableCodecs(
-        rtcSessionDescription: RTCSessionDescriptionInit
-    ): Array<string> {
+    parseAvailableCodecs(rtcSessionDescription: RTCSessionDescriptionInit): Array<string> {
         // No point in updating the available codecs if on FF
-        if (!RTCRtpReceiver.getCapabilities)
-            return ['Only available on Chrome'];
+        if (!RTCRtpReceiver.getCapabilities) return ['Only available on Chrome'];
 
         const ueSupportedCodecs: Array<string> = [];
         const sections = splitSections(rtcSessionDescription.sdp);