@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.10 → 0.0.11

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@epicgames-ps/lib-pixelstreamingfrontend-ue5.5",
-    "version": "0.0.10",
+    "version": "0.0.11",
     "description": "Frontend library for Unreal Engine 5.5 Pixel Streaming",
     "main": "dist/lib-pixelstreamingfrontend.js",
     "module": "dist/lib-pixelstreamingfrontend.esm.js",
@@ -16,7 +16,7 @@
         "spellcheck": "cspell \"{README.md,.github/*.md,src/**/*.ts}\""
     },
     "devDependencies": {
-        "@epicgames-ps/lib-pixelstreamingcommon-ue5.5": "^0.0.12",
+        "@epicgames-ps/lib-pixelstreamingcommon-ue5.5": "^0.0.13",
         "@types/jest": "27.5.1",
         "@types/webxr": "^0.5.1",
         "@typescript-eslint/eslint-plugin": "^6.21.0",
@@ -37,7 +37,7 @@
     },
     "repository": {
         "type": "git",
-        "url": "https://github.com/EpicGames/PixelStreamingInfrastructure.git"
+        "url": "https://github.com/EpicGamesExt/PixelStreamingInfrastructure.git"
     },
     "author": "Epic Games",
     "license": "MIT",

package/src/Config/Config.ts

@@ -31,7 +31,8 @@ export class Flags {
     static TouchInput = 'TouchInput' as const;
     static GamepadInput = 'GamepadInput' as const;
     static XRControllerInput = 'XRControllerInput' as const;
-    static WaitForStreamer = "WaitForStreamer" as const;
+    static WaitForStreamer = 'WaitForStreamer' as const;
+    static HideUI = 'HideUI' as const;
 }
 
 export type FlagsKeys = Exclude<keyof typeof Flags, 'prototype'>;
@@ -500,6 +501,19 @@ export class Config {
                 useUrlParams
             )
         );
+        
+        this.flags.set(
+            Flags.HideUI,
+            new SettingFlag(
+                Flags.HideUI,
+                'Hide the UI overlay',
+                'Will hide all UI overlay details',
+                settings && settings.hasOwnProperty(Flags.HideUI) ?
+                    settings[Flags.HideUI] :
+                    false,
+                useUrlParams
+            )
+        );
 
         /**
          * Numeric parameters

package/src/Config/SettingOption.ts

@@ -24,7 +24,7 @@ export class SettingOption<
 		// eslint-disable-next-line @typescript-eslint/no-empty-function
 		defaultOnChangeListener: (changedValue: unknown, setting: SettingBase) => void = () => { /* Do nothing, to be overridden. */ }
     ) {
-        super(id, label, description, [defaultTextValue], defaultOnChangeListener);
+        super(id, label, description, defaultTextValue, defaultOnChangeListener);
 
         this.options = options;
         const urlParams = new URLSearchParams(window.location.search);

package/src/Inputs/TouchController.ts

@@ -178,7 +178,7 @@ export class TouchController implements ITouchController {
                         coord.x,
                         coord.y,
                         this.fingerIds.get(touch.identifier),
-                        this.maxByteValue * touch.force,
+                        this.maxByteValue * (touch.force > 0 ? touch.force : 1),
                         coord.inRange ? 1 : 0
                     ]);
                     break;
@@ -198,7 +198,7 @@ export class TouchController implements ITouchController {
                         coord.x,
                         coord.y,
                         this.fingerIds.get(touch.identifier),
-                        this.maxByteValue * touch.force,
+                        this.maxByteValue * (touch.force > 0 ? touch.force : 1),
                         coord.inRange ? 1 : 0
                     ]);
                     break;

package/src/PeerConnectionController/AggregatedStats.ts

@@ -25,7 +25,7 @@ export class AggregatedStats {
     inboundAudioStats: InboundAudioStats;
     lastVideoStats: InboundVideoStats;
     lastAudioStats: InboundAudioStats;
-    candidatePair: CandidatePairStats;
+    candidatePairs: Array<CandidatePairStats>;
     DataChannelStats: DataChannelStats;
     localCandidates: Array<CandidateStat>;
     remoteCandidates: Array<CandidateStat>;
@@ -33,11 +33,11 @@ export class AggregatedStats {
     sessionStats: SessionStats;
     streamStats: StreamStats;
     codecs: Map<string, string>;
+    transportStats: RTCTransportStats;
 
     constructor() {
         this.inboundVideoStats = new InboundVideoStats();
         this.inboundAudioStats = new InboundAudioStats();
-        this.candidatePair = new CandidatePairStats();
         this.DataChannelStats = new DataChannelStats();
         this.outBoundVideoStats = new OutBoundVideoStats();
         this.sessionStats = new SessionStats();
@@ -52,6 +52,7 @@ export class AggregatedStats {
     processStats(rtcStatsReport: RTCStatsReport) {
         this.localCandidates = new Array<CandidateStat>();
         this.remoteCandidates = new Array<CandidateStat>();
+        this.candidatePairs = new Array<CandidatePairStats>();
 
         rtcStatsReport.forEach((stat) => {
             const type: RTCStatsTypePS = stat.type;
@@ -94,6 +95,7 @@ export class AggregatedStats {
                     this.handleTrack(stat);
                     break;
                 case 'transport':
+                    this.handleTransport(stat);
                     break;
                 case 'stream':
                     this.handleStream(stat);
@@ -120,16 +122,10 @@ export class AggregatedStats {
      * @param stat - the stats coming in from ice candidates
      */
     handleCandidatePair(stat: CandidatePairStats) {
-        this.candidatePair.bytesReceived = stat.bytesReceived;
-        this.candidatePair.bytesSent = stat.bytesSent;
-        this.candidatePair.localCandidateId = stat.localCandidateId;
-        this.candidatePair.remoteCandidateId = stat.remoteCandidateId;
-        this.candidatePair.nominated = stat.nominated;
-        this.candidatePair.readable = stat.readable;
-        this.candidatePair.selected = stat.selected;
-        this.candidatePair.writable = stat.writable;
-        this.candidatePair.state = stat.state;
-        this.candidatePair.currentRoundTripTime = stat.currentRoundTripTime;
+
+        // Add the candidate pair to the candidate pair array
+        this.candidatePairs.push(stat)
+
     }
 
     /**
@@ -162,6 +158,8 @@ export class AggregatedStats {
         localCandidate.protocol = stat.protocol;
         localCandidate.candidateType = stat.candidateType;
         localCandidate.id = stat.id;
+        localCandidate.relayProtocol = stat.relayProtocol;
+        localCandidate.transportId = stat.transportId;
         this.localCandidates.push(localCandidate);
     }
 
@@ -171,12 +169,14 @@ export class AggregatedStats {
      */
     handleRemoteCandidate(stat: CandidateStat) {
         const RemoteCandidate = new CandidateStat();
-        RemoteCandidate.label = 'local-candidate';
+        RemoteCandidate.label = 'remote-candidate';
         RemoteCandidate.address = stat.address;
         RemoteCandidate.port = stat.port;
         RemoteCandidate.protocol = stat.protocol;
         RemoteCandidate.id = stat.id;
         RemoteCandidate.candidateType = stat.candidateType;
+        RemoteCandidate.relayProtocol = stat.relayProtocol;
+        RemoteCandidate.transportId = stat.transportId
         this.remoteCandidates.push(RemoteCandidate);
     }
 
@@ -269,6 +269,11 @@ export class AggregatedStats {
         }
     }
 
+    handleTransport(stat: RTCTransportStats){
+        this.transportStats = stat;
+    }
+
+
     handleCodec(stat: CodecStats) {
         const codecId = stat.id;
         const codecType = `${stat.mimeType
@@ -308,4 +313,20 @@ export class AggregatedStats {
     isNumber(value: unknown): boolean {
         return typeof value === 'number' && isFinite(value);
     }
+
+    /**
+     * Helper function to return the active candidate pair
+     * @returns The candidate pair that is currently receiving data
+     */
+    public getActiveCandidatePair(): CandidatePairStats | null {
+        
+        // Check if the RTCTransport stat is not undefined
+        if (this.transportStats){
+            // Return the candidate pair that matches the transport candidate pair id
+            return this.candidatePairs.find((candidatePair) => candidatePair.id === this.transportStats.selectedCandidatePairId, null);
+        }
+        
+        // Fall back to the selected candidate pair
+        return this.candidatePairs.find((candidatePair) => candidatePair.selected, null);
+    }  
 }

package/src/PeerConnectionController/CandidatePairStats.ts

@@ -6,12 +6,19 @@
 export class CandidatePairStats {
     bytesReceived: number;
     bytesSent: number;
+    currentRoundTripTime: number;
+    id: string;
+    lastPacketReceivedTimestamp: number;
+    lastPacketSentTimestamp: number;
     localCandidateId: string;
-    remoteCandidateId: string;
     nominated: boolean;
+    priority: number;
     readable: boolean;
-    writable: boolean;
+    remoteCandidateId: string;
     selected: boolean;
     state: string;
-    currentRoundTripTime: number;
+    timestamp: number;
+    transportId: string;
+    type: string;
+    writable: boolean;  
 }

package/src/PeerConnectionController/CandidateStat.ts

@@ -4,10 +4,12 @@
  * ICE Candidate Stat collected from the RTC Stats Report
  */
 export class CandidateStat {
-    label: string;
-    id: string;
     address: string;
     candidateType: string;
+    id: string;
+    label: string;    
     port: number;
     protocol: 'tcp' | 'udp';
+    relayProtocol: 'tcp' | 'udp' | 'tls';
+    transportId: string;
 }

package/src/PixelStreaming/PixelStreaming.test.ts

@@ -400,9 +400,9 @@ describe('PixelStreaming', () => {
             expect.objectContaining({
                 data: {
                     aggregatedStats: expect.objectContaining({
-                        candidatePair: expect.objectContaining({
-                            bytesReceived: 123
-                        }),
+                        candidatePairs: [
+                            expect.objectContaining({ bytesReceived: 123 })
+                        ],
                         localCandidates: [
                             expect.objectContaining({ address: 'mock-address' })
                         ]

package/src/PixelStreaming/PixelStreaming.ts

@@ -28,7 +28,8 @@ import {
     WebRtcSdpEvent,
     DataChannelLatencyTestResponseEvent,
     DataChannelLatencyTestResultEvent,
-    PlayerCountEvent
+    PlayerCountEvent,
+    WebRtcTCPRelayDetectedEvent
 } from '../Util/EventEmitter';
 import { WebXRController } from '../WebXR/WebXRController';
 import { MessageDirection } from '../UeInstanceMessage/StreamMessageController';
@@ -61,6 +62,7 @@ export class PixelStreaming {
     protected _webRtcController: WebRtcPlayerController;
     protected _webXrController: WebXRController;
     protected _dataChannelLatencyTestController: DataChannelLatencyTestController;
+
     /**
      * Configuration object. You can read or modify config through this object. Whenever
      * the configuration is changed, the library will emit a `settingsChanged` event.
@@ -115,6 +117,15 @@ export class PixelStreaming {
             this.onScreenKeyboardHelper.showOnScreenKeyboard(command);
 
         this._webXrController = new WebXRController(this._webRtcController);
+
+        this._setupWebRtcTCPRelayDetection = this._setupWebRtcTCPRelayDetection.bind(this)
+
+        // Add event listener for the webRtcConnected event
+        this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => {
+
+            // Bind to the stats received event
+            this._eventEmitter.addEventListener("statsReceived",  this._setupWebRtcTCPRelayDetection);
+        });
     }
 
     /**
@@ -626,6 +637,28 @@ export class PixelStreaming {
         );
     }
 
+    // Sets up to emit the webrtc tcp relay detect event 
+    _setupWebRtcTCPRelayDetection(statsReceivedEvent: StatsReceivedEvent) {
+        // Get the active candidate pair
+        let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair();
+                
+        // Check if the active candidate pair is not null
+        if (activeCandidatePair != null) {
+
+            // Get the local candidate assigned to the active candidate pair
+            let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null)
+
+            // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp
+            if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') {
+
+                // Send the web rtc tcp relay detected event
+                this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent());
+            }
+            // The check is completed and the stats listen event can be removed
+            this._eventEmitter.removeEventListener("statsReceived", this._setupWebRtcTCPRelayDetection);
+        }
+    }
+
     /**
      * Request a connection latency test.
      * NOTE: There are plans to refactor all request* functions. Expect changes if you use this!

package/src/Util/EventEmitter.ts

@@ -537,6 +537,16 @@ export class PlayerCountEvent extends Event {
     }
 }
 
+/**
+ * An event that is emitted when the webRTC connections is relayed over TCP.
+ */
+export class WebRtcTCPRelayDetectedEvent extends Event {
+    readonly type: 'webRtcTCPRelayDetected';
+    constructor() {
+        super('webRtcTCPRelayDetected');
+    }
+}
+
 export type PixelStreamingEvent =
     | AfkWarningActivateEvent
     | AfkWarningUpdateEvent
@@ -573,7 +583,8 @@ export type PixelStreamingEvent =
     | XrSessionStartedEvent
     | XrSessionEndedEvent
     | XrFrameEvent
-    | PlayerCountEvent;
+    | PlayerCountEvent
+    | WebRtcTCPRelayDetectedEvent;
 
 export class EventEmitter extends EventTarget {
     /**

package/types/Config/Config.d.ts

@@ -27,6 +27,7 @@ export declare class Flags {
     static GamepadInput: "GamepadInput";
     static XRControllerInput: "XRControllerInput";
     static WaitForStreamer: "WaitForStreamer";
+    static HideUI: "HideUI";
 }
 export type FlagsKeys = Exclude<keyof typeof Flags, 'prototype'>;
 export type FlagsIds = typeof Flags[FlagsKeys];

package/types/PeerConnectionController/AggregatedStats.d.ts

@@ -12,7 +12,7 @@ export declare class AggregatedStats {
     inboundAudioStats: InboundAudioStats;
     lastVideoStats: InboundVideoStats;
     lastAudioStats: InboundAudioStats;
-    candidatePair: CandidatePairStats;
+    candidatePairs: Array<CandidatePairStats>;
     DataChannelStats: DataChannelStats;
     localCandidates: Array<CandidateStat>;
     remoteCandidates: Array<CandidateStat>;
@@ -20,6 +20,7 @@ export declare class AggregatedStats {
     sessionStats: SessionStats;
     streamStats: StreamStats;
     codecs: Map<string, string>;
+    transportStats: RTCTransportStats;
     constructor();
     /**
      * Gather all the information from the RTC Peer Connection Report
@@ -67,6 +68,7 @@ export declare class AggregatedStats {
      * @param stat - video track stats
      */
     handleTrack(stat: InboundTrackStats): void;
+    handleTransport(stat: RTCTransportStats): void;
     handleCodec(stat: CodecStats): void;
     handleSessionStatistics(videoStartTime: number, inputController: boolean | null, videoEncoderAvgQP: number): void;
     /**
@@ -74,4 +76,9 @@ export declare class AggregatedStats {
      * @param value - the number to be checked
      */
     isNumber(value: unknown): boolean;
+    /**
+     * Helper function to return the active candidate pair
+     * @returns The candidate pair that is currently receiving data
+     */
+    getActiveCandidatePair(): CandidatePairStats | null;
 }

package/types/PeerConnectionController/CandidatePairStats.d.ts

@@ -4,12 +4,19 @@
 export declare class CandidatePairStats {
     bytesReceived: number;
     bytesSent: number;
+    currentRoundTripTime: number;
+    id: string;
+    lastPacketReceivedTimestamp: number;
+    lastPacketSentTimestamp: number;
     localCandidateId: string;
-    remoteCandidateId: string;
     nominated: boolean;
+    priority: number;
     readable: boolean;
-    writable: boolean;
+    remoteCandidateId: string;
     selected: boolean;
     state: string;
-    currentRoundTripTime: number;
+    timestamp: number;
+    transportId: string;
+    type: string;
+    writable: boolean;
 }

package/types/PeerConnectionController/CandidateStat.d.ts

@@ -2,10 +2,12 @@
  * ICE Candidate Stat collected from the RTC Stats Report
  */
 export declare class CandidateStat {
-    label: string;
-    id: string;
     address: string;
     candidateType: string;
+    id: string;
+    label: string;
     port: number;
     protocol: 'tcp' | 'udp';
+    relayProtocol: 'tcp' | 'udp' | 'tls';
+    transportId: string;
 }

package/types/PixelStreaming/PixelStreaming.d.ts

@@ -3,7 +3,7 @@ import { LatencyTestResults } from '../DataChannel/LatencyTestResults';
 import { AggregatedStats } from '../PeerConnectionController/AggregatedStats';
 import { WebRtcPlayerController } from '../WebRtcPlayer/WebRtcPlayerController';
 import { InitialSettings } from '../DataChannel/InitialSettings';
-import { PixelStreamingEvent } from '../Util/EventEmitter';
+import { PixelStreamingEvent, StatsReceivedEvent } from '../Util/EventEmitter';
 import { WebXRController } from '../WebXR/WebXRController';
 import { MessageDirection } from '../UeInstanceMessage/StreamMessageController';
 import { DataChannelLatencyTestConfig, DataChannelLatencyTestController } from "../DataChannel/DataChannelLatencyTestController";
@@ -159,6 +159,7 @@ export declare class PixelStreaming {
      */
     _onQualityControlOwnership(hasQualityOwnership: boolean): void;
     _onPlayerCount(playerCount: number): void;
+    _setupWebRtcTCPRelayDetection(statsReceivedEvent: StatsReceivedEvent): void;
     /**
      * Request a connection latency test.
      * NOTE: There are plans to refactor all request* functions. Expect changes if you use this!

package/types/Util/EventEmitter.d.ts

@@ -395,7 +395,14 @@ export declare class PlayerCountEvent extends Event {
     };
     constructor(data: PlayerCountEvent['data']);
 }
-export type PixelStreamingEvent = AfkWarningActivateEvent | AfkWarningUpdateEvent | AfkWarningDeactivateEvent | AfkTimedOutEvent | VideoEncoderAvgQPEvent | WebRtcSdpEvent | WebRtcAutoConnectEvent | WebRtcConnectingEvent | WebRtcConnectedEvent | WebRtcFailedEvent | WebRtcDisconnectedEvent | DataChannelOpenEvent | DataChannelCloseEvent | DataChannelErrorEvent | VideoInitializedEvent | StreamLoadingEvent | StreamPreConnectEvent | StreamReconnectEvent | StreamPreDisconnectEvent | PlayStreamErrorEvent | PlayStreamEvent | PlayStreamRejectedEvent | LoadFreezeFrameEvent | HideFreezeFrameEvent | StatsReceivedEvent | StreamerListMessageEvent | StreamerIDChangedMessageEvent | LatencyTestResultEvent | DataChannelLatencyTestResponseEvent | DataChannelLatencyTestResultEvent | InitialSettingsEvent | SettingsChangedEvent | XrSessionStartedEvent | XrSessionEndedEvent | XrFrameEvent | PlayerCountEvent;
+/**
+ * An event that is emitted when the webRTC connections is relayed over TCP.
+ */
+export declare class WebRtcTCPRelayDetectedEvent extends Event {
+    readonly type: 'webRtcTCPRelayDetected';
+    constructor();
+}
+export type PixelStreamingEvent = AfkWarningActivateEvent | AfkWarningUpdateEvent | AfkWarningDeactivateEvent | AfkTimedOutEvent | VideoEncoderAvgQPEvent | WebRtcSdpEvent | WebRtcAutoConnectEvent | WebRtcConnectingEvent | WebRtcConnectedEvent | WebRtcFailedEvent | WebRtcDisconnectedEvent | DataChannelOpenEvent | DataChannelCloseEvent | DataChannelErrorEvent | VideoInitializedEvent | StreamLoadingEvent | StreamPreConnectEvent | StreamReconnectEvent | StreamPreDisconnectEvent | PlayStreamErrorEvent | PlayStreamEvent | PlayStreamRejectedEvent | LoadFreezeFrameEvent | HideFreezeFrameEvent | StatsReceivedEvent | StreamerListMessageEvent | StreamerIDChangedMessageEvent | LatencyTestResultEvent | DataChannelLatencyTestResponseEvent | DataChannelLatencyTestResultEvent | InitialSettingsEvent | SettingsChangedEvent | XrSessionStartedEvent | XrSessionEndedEvent | XrFrameEvent | PlayerCountEvent | WebRtcTCPRelayDetectedEvent;
 export declare class EventEmitter extends EventTarget {
     /**
      * Dispatch a new event.