@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.4.8 → 1.0.1

package/src/PeerConnectionController/OutBoundRTPStats.ts

@@ -1,26 +1,60 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 /**
- * Outbound Video Stats collected from the RTC Stats Report
+ * Outbound RTP stats collected from the RTC Stats Report under `outbound-rtp`.
+ * Wrapper around: https://developer.mozilla.org/en-US/docs/Web/API/RTCOutboundRtpStreamStats
+ * These are stats for video we are sending to a remote peer.
  */
-export class OutBoundVideoStats {
+export class OutboundRTPStats {
+    active: boolean | undefined;
+    codecId: string | undefined;
     bytesSent: number;
+    frameHeight: number | undefined;
+    frameWidth: number | undefined;
+    framesEncoded: number | undefined;
+    framesPerSecond: number | undefined;
+    framesSent: number | undefined;
+    headerBytesSent: number;
     id: string;
-    localId: string;
+    keyFramesEncoded: number | undefined;
+    kind: string;
+    mediaSourceId: string | undefined;
+    mid: string | undefined;
+    nackCount: number | undefined;
     packetsSent: number;
-    remoteTimestamp: number;
+    qpSum: number | undefined;
+    qualityLimitationDurations: number | undefined;
+    qualityLimitationReason: string | undefined;
+    remoteId: string | undefined;
+    retransmittedBytesSent: number;
+    rid: string | undefined;
+    scalabilityMode: string | undefined;
+    ssrc: string;
+    targetBitrate: number | undefined;
     timestamp: number;
+    totalEncodeTime: number | undefined;
+    totalEncodeBytesTarget: number | undefined;
+    totalPacketSendDelay: number | undefined;
+    transportId: string | undefined;
 }
 
 /**
- * Outbound Stats collected from the RTC Stats Report
+ * Remote outbound stats collected from the RTC Stats Report under `remote-outbound-rtp`.
+ * Wrapper around: https://developer.mozilla.org/en-US/docs/Web/API/RTCRemoteOutboundRtpStreamStats
+ * These are stats for media we are receiving from a remote peer.
  */
-export class OutBoundRTPStats {
+export class RemoteOutboundRTPStats {
+    bytesSent: number | undefined;
+    codecId: string;
+    id: string | undefined;
     kind: string;
-    bytesSent: number;
-    id: string;
-    localId: string;
-    packetsSent: number;
-    remoteTimestamp: number;
-    timestamp: number;
+    localId: string | undefined;
+    packetsSent: number | undefined;
+    remoteTimestamp: number | undefined;
+    reportsSent: number | undefined;
+    roundTripTimeMeasurements: number | undefined;
+    ssrc: string;
+    timestamp: number | undefined;
+    totalRoundTripTime: number | undefined;
+    transportId: string | undefined;
 }

package/src/PeerConnectionController/PeerConnectionController.ts

@@ -6,6 +6,10 @@ import { AggregatedStats } from './AggregatedStats';
 import { parseRtpParameters, splitSections } from 'sdp';
 import { RTCUtils } from '../Util/RTCUtils';
 import { CodecStats } from './CodecStats';
+import { SDPUtils } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { LatencyCalculator, LatencyInfo } from './LatencyCalculator';
+
+export const kAbsCaptureTime = 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time';
 
 /**
  * Handles the Peer Connection
@@ -18,6 +22,7 @@ export class PeerConnectionController {
     updateCodecSelection: boolean;
     videoTrack: MediaStreamTrack;
     audioTrack: MediaStreamTrack;
+    latencyCalculator: LatencyCalculator;
 
     /**
      * Create a new RTC Peer Connection client
@@ -27,6 +32,7 @@ export class PeerConnectionController {
     constructor(options: RTCConfiguration, config: Config, preferredCodec: string) {
         this.config = config;
         this.createPeerConnection(options, preferredCodec);
+        this.latencyCalculator = new LatencyCalculator();
     }
 
     createPeerConnection(options: RTCConfiguration, preferredCodec: string) {
@@ -88,12 +94,26 @@ export class PeerConnectionController {
     }
 
     /**
-     *
+     * Receive offer from UE side and process it as the remote description of this peer connection
      */
     async receiveOffer(offer: RTCSessionDescriptionInit, config: Config) {
         Logger.Info('Receive Offer');
 
+        // If UE or JSStreamer did send abs-capture-time RTP header extension to a non-Chrome browser
+        // then remove it from the SDP because if Firefox detects it in offer or answer it will fail to connect
+        // due having 15 or more header extensions: https://mailarchive.ietf.org/arch/msg/rtcweb/QRnWNuWzGuLRovWdHkodNP6VOgg/
+        if (this.isFirefox()) {
+            // example: a=extmap:15 http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
+            offer.sdp = offer.sdp.replace(
+                /^a=extmap:\d+ http:\/\/www\.webrtc\.org\/experiments\/rtp-hdrext\/abs-capture-time\r\n/gm,
+                ''
+            );
+        }
+
         this.peerConnection?.setRemoteDescription(offer).then(() => {
+            // Fire event for when remote offer description is set
+            this.onSetRemoteDescription(offer);
+
             const isLocalhostConnection =
                 location.hostname === 'localhost' || location.hostname === '127.0.0.1';
             const isHttpsConnection = location.protocol === 'https:';
@@ -124,10 +144,10 @@ export class PeerConnectionController {
                         return this.peerConnection?.setLocalDescription(Answer);
                     })
                     .then(() => {
-                        this.onSendWebRTCAnswer(this.peerConnection?.currentLocalDescription);
+                        this.onSetLocalDescription(this.peerConnection?.currentLocalDescription);
                     })
-                    .catch(() => {
-                        Logger.Error('createAnswer() failed');
+                    .catch((err) => {
+                        Logger.Error(`createAnswer() failed - ${err}`);
                     });
             });
         });
@@ -151,25 +171,29 @@ export class PeerConnectionController {
      * Generate Aggregated Stats and then fire a onVideo Stats event
      */
     generateStats() {
-        const audioPromise = this.audioTrack
-            ? this.peerConnection?.getStats(this.audioTrack).then((statsData: RTCStatsReport) => {
-                  this.aggregatedStats.processStats(statsData);
-              })
-            : Promise.resolve();
-        const videoPromise = this.videoTrack
-            ? this.peerConnection?.getStats(this.videoTrack).then((statsData: RTCStatsReport) => {
-                  this.aggregatedStats.processStats(statsData);
-              })
-            : Promise.resolve();
-
-        Promise.allSettled([audioPromise, videoPromise]).then(() => {
+        this.peerConnection.getStats().then((statsData: RTCStatsReport) => {
+            this.aggregatedStats.processStats(statsData);
+
             this.onVideoStats(this.aggregatedStats);
+
+            // Calculate latency using stats and video receivers and then call the handling function
+            const latencyInfo: LatencyInfo = this.latencyCalculator.calculate(
+                this.aggregatedStats,
+                this.peerConnection.getReceivers()
+            );
+            this.onLatencyCalculated(latencyInfo);
+
             // Update the preferred codec selection based on what was actually negotiated
             if (this.updateCodecSelection && !!this.aggregatedStats.inboundVideoStats.codecId) {
                 // Construct the qualified codec name from the mimetype and fmtp
-                const codecStats: CodecStats = this.aggregatedStats.codecs.get(
+                const codecStats: CodecStats | undefined = this.aggregatedStats.codecs.get(
                     this.aggregatedStats.inboundVideoStats.codecId
                 );
+
+                if (codecStats === undefined) {
+                    return;
+                }
+
                 const codecShortname = codecStats.mimeType.replace('video/', '');
                 let fullCodecName = codecShortname;
                 if (codecStats.sdpFmtpLine && codecStats.sdpFmtpLine.trim() !== '') {
@@ -237,9 +261,20 @@ export class PeerConnectionController {
         // We use the line 'useinbandfec=1' (which Opus uses) to set our Opus specific audio parameters.
         mungedSDP = mungedSDP.replace('useinbandfec=1', audioSDP);
 
+        // Add abs-capture-time RTP header extension if we have enabled the setting.
+        // Note: As at Feb 2025, Chromium based browsers are the only ones that support this and
+        // munging it into the answer in Firefox will cause the connection to fail.
+        if (this.config.isFlagEnabled(Flags.EnableCaptureTimeExt) && !this.isFirefox()) {
+            mungedSDP = SDPUtils.addVideoHeaderExtensionToSdp(mungedSDP, kAbsCaptureTime);
+        }
+
         return mungedSDP;
     }
 
+    isFirefox(): boolean {
+        return navigator.userAgent.indexOf('Firefox') > 0;
+    }
+
     /**
      * When a Ice Candidate is received add to the RTC Peer Connection
      * @param iceCandidate - RTC Ice Candidate from the Signaling Server
@@ -586,6 +621,15 @@ export class PeerConnectionController {
         // Default Functionality: Do Nothing
     }
 
+    /**
+     * And override event for when latency info is calculated
+     * @param latencyInfo - Calculated latency information.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    onLatencyCalculated(latencyInfo: LatencyInfo) {
+        // Default Functionality: Do Nothing
+    }
+
     /**
      * Event to send the RTC offer to the Signaling server
      * @param offer - RTC Offer
@@ -596,11 +640,20 @@ export class PeerConnectionController {
     }
 
     /**
-     * Event to send the RTC Answer to the Signaling server
+     * Event fired when remote offer description is set.
+     * @param offer - RTC Offer
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    onSetRemoteDescription(offer: RTCSessionDescriptionInit) {
+        // Default Functionality: Do Nothing
+    }
+
+    /**
+     * Event fire when local description answer is set.
      * @param answer - RTC Answer
      */
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    onSendWebRTCAnswer(answer: RTCSessionDescriptionInit) {
+    onSetLocalDescription(answer: RTCSessionDescriptionInit) {
         // Default Functionality: Do Nothing
     }
 

package/src/PixelStreaming/PixelStreaming.ts

@@ -11,6 +11,7 @@ import { OnScreenKeyboard } from '../UI/OnScreenKeyboard';
 import {
     PixelStreamingEventEmitter,
     InitialSettingsEvent,
+    LatencyCalculatedEvent,
     LatencyTestResultEvent,
     PixelStreamingEvent,
     StatsReceivedEvent,
@@ -30,7 +31,9 @@ import {
     DataChannelLatencyTestResultEvent,
     PlayerCountEvent,
     WebRtcTCPRelayDetectedEvent,
-    SubscribeFailedEvent
+    SubscribeFailedEvent,
+    WebRtcSdpOfferEvent,
+    WebRtcSdpAnswerEvent
 } from '../Util/EventEmitter';
 import { WebXRController } from '../WebXR/WebXRController';
 import { MessageDirection } from '../UeInstanceMessage/StreamMessageController';
@@ -44,6 +47,7 @@ import {
 } from '../DataChannel/DataChannelLatencyTestResults';
 import { RTCUtils } from '../Util/RTCUtils';
 import { IURLSearchParams } from '../Util/IURLSearchParams';
+import { LatencyInfo } from '../PeerConnectionController/LatencyCalculator';
 
 export interface PixelStreamingOverrides {
     /** The DOM element where Pixel Streaming video and user input event handlers are attached to.
@@ -450,21 +454,42 @@ export class PixelStreaming {
     }
 
     /**
-     * Emit an event on auto connecting
+     * Internal function to emit an event when auto connecting occurs
      */
     _onWebRtcAutoConnect() {
         this._eventEmitter.dispatchEvent(new WebRtcAutoConnectEvent());
     }
 
     /**
-     * Set up functionality to happen when receiving a webRTC answer
+     * Internal function to emit an event for when SDP negotiation is fully finished.
      */
     _onWebRtcSdp() {
         this._eventEmitter.dispatchEvent(new WebRtcSdpEvent());
     }
 
     /**
-     * Emits a StreamLoading event
+     * Internal function to emit an SDP offer after it has been set.
+     */
+    _onWebRtcSdpOffer(offer: RTCSessionDescriptionInit) {
+        this._eventEmitter.dispatchEvent(new WebRtcSdpOfferEvent({ sdp: offer }));
+    }
+
+    /**
+     * Internal function to emit an SDP answer after it has been set.
+     */
+    _onWebRtcSdpAnswer(answer: RTCSessionDescriptionInit) {
+        this._eventEmitter.dispatchEvent(new WebRtcSdpAnswerEvent({ sdp: answer }));
+    }
+
+    /**
+     * Internal function call to emit a `latencyCalculated` event.
+     */
+    _onLatencyCalculated(latencyInfo: LatencyInfo) {
+        this._eventEmitter.dispatchEvent(new LatencyCalculatedEvent({ latencyInfo }));
+    }
+
+    /**
+     * Internal function to emits a StreamLoading event
      */
     _onStreamLoading() {
         this._eventEmitter.dispatchEvent(new StreamLoadingEvent());

package/src/Util/EventEmitter.ts

@@ -2,6 +2,7 @@ import { FlagsIds, NumericParametersIds, OptionParametersIds, TextParametersIds
 import { LatencyTestResults } from '../DataChannel/LatencyTestResults';
 import { AggregatedStats } from '../PeerConnectionController/AggregatedStats';
 import { InitialSettings } from '../DataChannel/InitialSettings';
+import { LatencyInfo } from '../PeerConnectionController/LatencyCalculator';
 import { Messages } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
 import { SettingFlag } from '../Config/SettingFlag';
 import { SettingNumber } from '../Config/SettingNumber';
@@ -90,6 +91,36 @@ export class WebRtcSdpEvent extends Event {
     }
 }
 
+/**
+ * An event that is emitted after the SDP answer is set.
+ */
+export class WebRtcSdpAnswerEvent extends Event {
+    override readonly type: 'webRtcSdpAnswer';
+    readonly data: {
+        /** The sdp answer */
+        sdp: RTCSessionDescriptionInit;
+    };
+    constructor(data: WebRtcSdpAnswerEvent['data']) {
+        super('webRtcSdpAnswer');
+        this.data = data;
+    }
+}
+
+/**
+ * An event that is emitted after the SDP offer is set.
+ */
+export class WebRtcSdpOfferEvent extends Event {
+    override readonly type: 'webRtcSdpOffer';
+    readonly data: {
+        /** The sdp offer */
+        sdp: RTCSessionDescriptionInit;
+    };
+    constructor(data: WebRtcSdpOfferEvent['data']) {
+        super('webRtcSdpOffer');
+        this.data = data;
+    }
+}
+
 /**
  * An event that is emitted when auto connecting.
  */
@@ -382,6 +413,20 @@ export class LatencyTestResultEvent extends Event {
     }
 }
 
+/**
+ * An event that is emitted everytime latency is calculated using the WebRTC stats API.
+ */
+export class LatencyCalculatedEvent extends Event {
+    override readonly type: 'latencyCalculated';
+    readonly data: {
+        latencyInfo: LatencyInfo;
+    };
+    constructor(data: LatencyCalculatedEvent['data']) {
+        super('latencyCalculated');
+        this.data = data;
+    }
+}
+
 /**
  * An event that is emitted when receiving data channel latency test response from server.
  * This event is handled by DataChannelLatencyTestController
@@ -560,6 +605,8 @@ export type PixelStreamingEvent =
     | AfkTimedOutEvent
     | VideoEncoderAvgQPEvent
     | WebRtcSdpEvent
+    | WebRtcSdpOfferEvent
+    | WebRtcSdpAnswerEvent
     | WebRtcAutoConnectEvent
     | WebRtcConnectingEvent
     | WebRtcConnectedEvent
@@ -581,6 +628,7 @@ export type PixelStreamingEvent =
     | StatsReceivedEvent
     | StreamerListMessageEvent
     | StreamerIDChangedMessageEvent
+    | LatencyCalculatedEvent
     | LatencyTestResultEvent
     | DataChannelLatencyTestResponseEvent
     | DataChannelLatencyTestResultEvent

package/src/VideoPlayer/VideoPlayer.ts

@@ -8,7 +8,7 @@ import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
  */
 declare global {
     interface HTMLElement {
-        mozRequestPointerLock(options: PointerLockOptions): Promise<void>;
+        mozRequestPointerLock(): Promise<void>;
     }
 }
 

package/src/WebRtcPlayer/WebRtcPlayerController.ts

@@ -55,6 +55,7 @@ import {
 import { IURLSearchParams } from '../Util/IURLSearchParams';
 import { IInputController } from '../Inputs/IInputController';
 import { GamepadController } from '../Inputs/GamepadController';
+import { LatencyInfo } from '../PeerConnectionController/LatencyCalculator';
 
 /**
  * Entry point for the WebRTC Player
@@ -1075,15 +1076,29 @@ export class WebRtcPlayerController {
         );
 
         // set up peer connection controller video stats
-        this.peerConnectionController.onVideoStats = (event: AggregatedStats) => this.handleVideoStats(event);
+        this.peerConnectionController.onVideoStats = (event: AggregatedStats) => {
+            this.handleVideoStats(event);
+        };
+
+        /* Set event handler for latency information is calculated, handle the event by propogating to the PixelStreaming API */
+        this.peerConnectionController.onLatencyCalculated = (latencyInfo: LatencyInfo) => {
+            this.pixelStreaming._onLatencyCalculated(latencyInfo);
+        };
 
         /* When the Peer Connection wants to send an offer have it handled */
-        this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) =>
+        this.peerConnectionController.onSendWebRTCOffer = (offer: RTCSessionDescriptionInit) => {
             this.handleSendWebRTCOffer(offer);
+        };
 
-        /* When the Peer Connection wants to send an answer have it handled */
-        this.peerConnectionController.onSendWebRTCAnswer = (offer: RTCSessionDescriptionInit) =>
-            this.handleSendWebRTCAnswer(offer);
+        /* Set event handler for when local answer description is set */
+        this.peerConnectionController.onSetLocalDescription = (answer: RTCSessionDescriptionInit) => {
+            this.handleSendWebRTCAnswer(answer);
+        };
+
+        /* Set event handler for when remote offer description is set */
+        this.peerConnectionController.onSetRemoteDescription = (offer: RTCSessionDescriptionInit) => {
+            this.pixelStreaming._onWebRtcSdpOffer(offer);
+        };
 
         /* When the Peer Connection ice candidate is added have it handled */
         this.peerConnectionController.onPeerIceCandidate = (
@@ -1484,6 +1499,9 @@ export class WebRtcPlayerController {
         if (this.isUsingSFU) {
             this.protocol.sendMessage(MessageHelpers.createMessage(Messages.dataChannelRequest));
         }
+
+        // Send answer back to Pixel Streaming main class for event dispatch
+        this.pixelStreaming._onWebRtcSdpAnswer(answer);
     }
 
     /**

package/src/__test__/mockRTCPeerConnection.ts

@@ -111,7 +111,7 @@ export class MockRTCPeerConnectionImpl implements RTCPeerConnection {
         throw new Error("Method not implemented.");
     }
     getReceivers(): RTCRtpReceiver[] {
-        throw new Error("Method not implemented.");
+        return [];
     }
     getSenders(): RTCRtpSender[] {
         throw new Error("Method not implemented.");

package/src/pixelstreamingfrontend.ts

@@ -19,7 +19,8 @@ export { CandidatePairStats } from './PeerConnectionController/CandidatePairStat
 export { CandidateStat } from './PeerConnectionController/CandidateStat';
 export { DataChannelStats } from './PeerConnectionController/DataChannelStats';
 export { InboundAudioStats, InboundVideoStats } from './PeerConnectionController/InboundRTPStats';
-export { OutBoundVideoStats } from './PeerConnectionController/OutBoundRTPStats';
+export { OutboundRTPStats, RemoteOutboundRTPStats } from './PeerConnectionController/OutBoundRTPStats';
+export * from './PeerConnectionController/LatencyCalculator';
 export * from './DataChannel/DataChannelLatencyTestResults';
 export * from './Util/EventEmitter';
 export * from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';

package/tsconfig.base.json

@@ -15,8 +15,8 @@
         "declarationDir": "./dist/types"
     },
     "lib": ["ES6"],
-    "include": ["./src/*.ts"],
-    "exclude": ["./src/**/*.test.ts", "./coverage/**", "./dist/**"],
+    "include": ["./src/**/*.ts"],
+    "exclude": ["./src/**/*.test.ts"],
     "typedocOptions": {
         "exclude": "src/index.*",
         "entryPoints": ["src/pixelstreamingfrontend.ts"],

package/.eslintignore

@@ -1,12 +0,0 @@
-dist
-node_modules
-types
-package-lock.json
-package.json
-tsconfig.json
-*.config.js
-.eslintrc.js
-src/__test__/
-webpack.*.js
-*.test.ts
-coverage

package/.eslintrc.js

@@ -1,20 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-module.exports = {
-    root: true,
-    parser: '@typescript-eslint/parser',
-    parserOptions: { project: './tsconfig.cjs.json' },
-    plugins: ['@typescript-eslint'],
-    extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', "plugin:prettier/recommended"],
-    rules: {
-        "@typescript-eslint/no-explicit-any": "off",
-        "@typescript-eslint/no-unused-vars": [
-            "error",
-            {
-                "argsIgnorePattern": "^_",
-                "varsIgnorePattern": "^_",
-                "caughtErrorsIgnorePattern": "^_"
-            }
-        ]
-    }
-};

package/.prettierrc.json

@@ -1,7 +0,0 @@
-{
-    "printWidth": 110,
-    "trailingComma": "none",
-    "tabWidth": 4,
-    "semi": true,
-    "singleQuote": true
-}