@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.11 → 0.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@epicgames-ps/lib-pixelstreamingfrontend-ue5.5",
-    "version": "0.0.11",
+    "version": "0.0.12",
     "description": "Frontend library for Unreal Engine 5.5 Pixel Streaming",
     "main": "dist/lib-pixelstreamingfrontend.js",
     "module": "dist/lib-pixelstreamingfrontend.esm.js",
@@ -16,7 +16,7 @@
         "spellcheck": "cspell \"{README.md,.github/*.md,src/**/*.ts}\""
     },
     "devDependencies": {
-        "@epicgames-ps/lib-pixelstreamingcommon-ue5.5": "^0.0.13",
+        "@epicgames-ps/lib-pixelstreamingcommon-ue5.5": "^0.0.14",
         "@types/jest": "27.5.1",
         "@types/webxr": "^0.5.1",
         "@typescript-eslint/eslint-plugin": "^6.21.0",
@@ -45,3 +45,4 @@
         "access": "public"
     }
 }
+

package/src/Config/Config.ts

@@ -508,7 +508,7 @@ export class Config {
                 Flags.HideUI,
                 'Hide the UI overlay',
                 'Will hide all UI overlay details',
-                settings && settings.hasOwnProperty(Flags.HideUI) ?
+                settings && Object.prototype.hasOwnProperty.call(settings, Flags.HideUI) ?
                     settings[Flags.HideUI] :
                     false,
                 useUrlParams

package/src/PeerConnectionController/PeerConnectionController.ts

@@ -15,6 +15,8 @@ export class PeerConnectionController {
     config: Config;
     preferredCodec: string;
     updateCodecSelection: boolean;
+    videoTrack: MediaStreamTrack;
+    audioTrack: MediaStreamTrack;
 
     /**
      * Create a new RTC Peer Connection client
@@ -175,10 +177,15 @@ export class PeerConnectionController {
      * Generate Aggregated Stats and then fire a onVideo Stats event
      */
     generateStats() {
-        this.peerConnection?.getStats(null).then((StatsData: RTCStatsReport) => {
+        const statsHandler = (StatsData: RTCStatsReport) => {
             this.aggregatedStats.processStats(StatsData);
-            this.onVideoStats(this.aggregatedStats);
+        };
+
+        const audioPromise = this.peerConnection?.getStats(this.audioTrack).then(statsHandler);
+        const videoPromise = this.peerConnection?.getStats(this.videoTrack).then(statsHandler);
 
+        Promise.allSettled([audioPromise, videoPromise]).then(() => {
+            this.onVideoStats(this.aggregatedStats);
             // Update the preferred codec selection based on what was actually negotiated
             if (this.updateCodecSelection && !!this.aggregatedStats.inboundVideoStats.codecId) {
                 this.config.setOptionSettingValue(
@@ -300,6 +307,15 @@ export class PeerConnectionController {
      * @param event - The webRtc track event
      */
     handleOnTrack(event: RTCTrackEvent) {
+        if (event.streams.length < 1 || event.streams[0].id == 'probator') {
+            return;
+        }
+        if (event.track.kind == 'video') {
+            this.videoTrack = event.track;
+        }
+        if (event.track.kind == 'audio') {
+            this.audioTrack = event.track;
+        }
         this.onTrack(event);
     }
 

package/src/PixelStreaming/PixelStreaming.test.ts

@@ -6,7 +6,7 @@ import {
 import { PixelStreaming } from './PixelStreaming';
 import { SettingsChangedEvent, StreamerListMessageEvent, WebRtcConnectedEvent, WebRtcSdpEvent } from '../Util/EventEmitter';
 import { mockWebSocket, MockWebSocketSpyFunctions, MockWebSocketTriggerFunctions, unmockWebSocket } from '../__test__/mockWebSocket';
-import { MessageReceive } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { BaseMessage, Messages, MessageHelpers } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
 import { mockRTCPeerConnection, MockRTCPeerConnectionSpyFunctions, MockRTCPeerConnectionTriggerFunctions, unmockRTCPeerConnection } from '../__test__/mockRTCPeerConnection';
 import { mockHTMLMediaElement, mockMediaStream, unmockMediaStream } from '../__test__/mockMediaStream';
 import { InitialSettings } from '../DataChannel/InitialSettings';
@@ -32,26 +32,13 @@ describe('PixelStreaming', () => {
 
     const triggerWebSocketOpen = () =>
         webSocketTriggerFunctions.triggerOnOpen?.();
-    const triggerConfigMessage = () =>
-        webSocketTriggerFunctions.triggerOnMessage?.({
-            type: MessageReceive.MessageRecvTypes.CONFIG,
-            peerConnectionOptions: {}
-        });
-    const triggerStreamerListMessage = (streamerIdList: string[]) =>
-        webSocketTriggerFunctions.triggerOnMessage?.({
-            type: MessageReceive.MessageRecvTypes.STREAMER_LIST,
-            ids: streamerIdList
-        });
-    const triggerSdpOfferMessage = () =>
-        webSocketTriggerFunctions.triggerOnMessage?.({
-            type: MessageReceive.MessageRecvTypes.OFFER,
-            sdp
-        });
-    const triggerIceCandidateMessage = () =>
-        webSocketTriggerFunctions.triggerOnMessage?.({
-            type: MessageReceive.MessageRecvTypes.ICE_CANDIDATE,
-            candidate: iceCandidate
-        });
+    const triggerSignallingMessage = (message: BaseMessage) => {
+        webSocketTriggerFunctions.triggerOnMessage?.(message);
+    }
+    const triggerConfigMessage = () => triggerSignallingMessage(MessageHelpers.createMessage(Messages.config, { peerConnectionOptions: {} }));
+    const triggerStreamerListMessage = (streamerIdList: string[]) => triggerSignallingMessage(MessageHelpers.createMessage(Messages.streamerList, { ids: streamerIdList }));
+    const triggerSdpOfferMessage = () => triggerSignallingMessage(MessageHelpers.createMessage(Messages.offer, { sdp }));
+    const triggerIceCandidateMessage = () => triggerSignallingMessage(MessageHelpers.createMessage(Messages.iceCandidate, { candidate: iceCandidate }));
     const triggerIceConnectionState = (state: RTCIceConnectionState) =>
         rtcPeerConnectionTriggerFunctions.triggerIceConnectionStateChange?.(
             state
@@ -200,17 +187,13 @@ describe('PixelStreaming', () => {
 
     it('should automatically reconnect and request streamer list N times on websocket close', () => {
         const config = new Config({ initialSettings: {ss: mockSignallingUrl, AutoConnect: true, MaxReconnectAttempts: 3}});
-        const autoconnectedSpy = jest.fn();
-
         const pixelStreaming = new PixelStreaming(config);
-        pixelStreaming.addEventListener("webRtcAutoConnect", autoconnectedSpy);
 
         expect(webSocketSpyFunctions.constructorSpy).toHaveBeenCalledWith(mockSignallingUrl);
         expect(webSocketSpyFunctions.constructorSpy).toHaveBeenCalledTimes(1);
         expect(webSocketSpyFunctions.closeSpy).not.toHaveBeenCalled();
 
-    
-        webSocketTriggerFunctions.triggerRemoteClose();
+        webSocketTriggerFunctions.triggerRemoteClose?.();
 
         expect(webSocketSpyFunctions.closeSpy).toHaveBeenCalled();
 
@@ -273,11 +256,11 @@ describe('PixelStreaming', () => {
         
         expect(streamerListSpy).toHaveBeenCalledWith(new StreamerListMessageEvent({
             messageStreamerList: expect.objectContaining({
-                type: MessageReceive.MessageRecvTypes.STREAMER_LIST,
+                type: Messages.streamerList.typeName,
                 ids: streamerIdList
             }),
             autoSelectedStreamerId: streamerId,
-            wantedStreamerId: null
+            wantedStreamerId: '' 
         }));
         expect(webSocketSpyFunctions.sendSpy).toHaveBeenCalledWith(
             expect.stringMatching(/"type":"subscribe".*MOCK_PIXEL_STREAMING/)
@@ -298,11 +281,11 @@ describe('PixelStreaming', () => {
         
         expect(streamerListSpy).toHaveBeenCalledWith(new StreamerListMessageEvent({
             messageStreamerList: expect.objectContaining({
-                type: MessageReceive.MessageRecvTypes.STREAMER_LIST,
+                type: Messages.streamerList.typeName,
                 ids: extendedStreamerIdList
             }),
-            autoSelectedStreamerId: null,
-            wantedStreamerId: null
+            autoSelectedStreamerId: '',
+            wantedStreamerId: ''
         }));
         expect(webSocketSpyFunctions.sendSpy).not.toHaveBeenCalledWith(
             expect.stringMatching(/"type":"subscribe"/)
@@ -443,22 +426,22 @@ describe('PixelStreaming', () => {
         expect(streamSpy).toHaveBeenCalled();
     });
 
-    it('should emit playStreamRejected if video play is rejected', async () => {
-        mockHTMLMediaElement({ ableToPlay: false });
-        
-        const config = new Config({ initialSettings: {ss: mockSignallingUrl}});
-        const streamRejectedSpy = jest.fn();
-        const pixelStreaming = new PixelStreaming(config);
-        pixelStreaming.addEventListener("playStreamRejected", streamRejectedSpy);
-        pixelStreaming.connect();
-
-        establishMockedPixelStreamingConnection();
-
-        pixelStreaming.play();
-        await flushPromises();
-
-        expect(streamRejectedSpy).toHaveBeenCalled();
-    });
+    // it('should emit playStreamRejected if video play is rejected', async () => {
+    //     mockHTMLMediaElement({ ableToPlay: false });
+    //     
+    //     const config = new Config({ initialSettings: {ss: mockSignallingUrl}});
+    //     const streamRejectedSpy = jest.fn();
+    //     const pixelStreaming = new PixelStreaming(config);
+    //     pixelStreaming.addEventListener("playStreamRejected", streamRejectedSpy);
+    //     pixelStreaming.connect();
+    //
+    //     establishMockedPixelStreamingConnection();
+    //
+    //     pixelStreaming.play();
+    //     await flushPromises();
+    //
+    //     expect(streamRejectedSpy).toHaveBeenCalled();
+    // });
 
     it('should send data through the data channel when emitCommand is called', () => {
         mockHTMLMediaElement({ ableToPlay: true, readyState: 2 });

package/src/PixelStreaming/PixelStreaming.ts

@@ -121,7 +121,7 @@ export class PixelStreaming {
         this._setupWebRtcTCPRelayDetection = this._setupWebRtcTCPRelayDetection.bind(this)
 
         // Add event listener for the webRtcConnected event
-        this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => {
+        this._eventEmitter.addEventListener("webRtcConnected", (_: WebRtcConnectedEvent) => {
 
             // Bind to the stats received event
             this._eventEmitter.addEventListener("statsReceived",  this._setupWebRtcTCPRelayDetection);
@@ -640,13 +640,13 @@ export class PixelStreaming {
     // Sets up to emit the webrtc tcp relay detect event 
     _setupWebRtcTCPRelayDetection(statsReceivedEvent: StatsReceivedEvent) {
         // Get the active candidate pair
-        let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair();
+        const activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair();
                 
         // Check if the active candidate pair is not null
         if (activeCandidatePair != null) {
 
             // Get the local candidate assigned to the active candidate pair
-            let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null)
+            const localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null)
 
             // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp
             if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') {

package/src/VideoPlayer/StreamController.ts

@@ -31,6 +31,12 @@ export class StreamController {
             'handleOnTrack ' + JSON.stringify(rtcTrackEvent.streams),
             6
         );
+        // Do not add the track if the ID is `probator` as this is special track created by mediasoup for bitrate probing.
+        // Refer to https://github.com/EpicGamesExt/PixelStreamingInfrastructure/pull/86 for more details.
+        if (rtcTrackEvent.streams.length < 1 || rtcTrackEvent.streams[0].id == 'probator') {
+            return;
+        }
+
         const videoElement = this.videoElementProvider.getVideoElement();
 
         if (rtcTrackEvent.track) {

package/src/WebRtcPlayer/WebRtcPlayerController.ts

@@ -222,6 +222,7 @@ export class WebRtcPlayerController {
                 const message = MessageHelpers.createMessage(Messages.listStreamers);
                 this.protocol.sendMessage(message);
             }
+            this.reconnectAttempt = 0;
         });
         this.protocol.transport.addListener('error', () => {
             // dont really need to do anything here since the close event should follow.
@@ -1324,7 +1325,7 @@ export class WebRtcPlayerController {
             6
         );
 
-        let wantedStreamerId: string = null;
+        let wantedStreamerId: string = '';
 
         // get the current selected streamer id option
         const streamerIDOption = this.config.getSettingOption(OptionParameters.StreamerId);
@@ -1342,7 +1343,7 @@ export class WebRtcPlayerController {
             settingOptions
         );
 
-        let autoSelectedStreamerId: string = null;
+        let autoSelectedStreamerId: string = '';
         const waitForStreamer = this.config.isFlagEnabled(Flags.WaitForStreamer);
         const reconnectLimit = this.config.getNumericSettingValue(NumericParameters.MaxReconnectAttempts);
         const reconnectDelay = this.config.getNumericSettingValue(NumericParameters.StreamerAutoJoinInterval);

package/src/__test__/mockWebSocket.ts

@@ -1,3 +1,5 @@
+import { BaseMessage } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+
 export interface MockWebSocketSpyFunctions {
     constructorSpy: null | ((url: string) => void);
     openSpy: null | ((event: Event) => void);
@@ -12,7 +14,7 @@ export interface MockWebSocketTriggerFunctions {
     triggerOnOpen: null | (() => void);
     triggerOnError: null | (() => void);
     triggerOnClose: null | ((closeReason?: CloseEventInit) => void);
-    triggerOnMessage: null | ((message?: object) => void);
+    triggerOnMessage: null | ((message?: BaseMessage) => void);
     triggerOnMessageBinary: null | ((message?: Blob) => void);
     triggerRemoteClose: null | ((code?: number, reason?: string) => void);
 }
@@ -88,10 +90,8 @@ export class MockWebSocketImpl extends WebSocket {
         this.close(code, reason);
     }
 
-    triggerOnMessage(message?: object) {
-        const data = message
-            ? JSON.stringify(message)
-            : JSON.stringify({ type: 'test' });
+    triggerOnMessage(message: BaseMessage) {
+        const data = JSON.stringify(message);
         const event = new MessageEvent('message', { data });
         this.onmessage?.(event);
         spyFunctions.messageSpy?.(event);

package/types/PeerConnectionController/PeerConnectionController.d.ts

@@ -9,6 +9,8 @@ export declare class PeerConnectionController {
     config: Config;
     preferredCodec: string;
     updateCodecSelection: boolean;
+    videoTrack: MediaStreamTrack;
+    audioTrack: MediaStreamTrack;
     /**
      * Create a new RTC Peer Connection client
      * @param options - Peer connection Options