@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.5

package/src/Inputs/XRGamepadController.ts

@@ -0,0 +1,126 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController';
+import { Controller } from './GamepadTypes';
+import { WebXRUtils } from '../Util/WebXRUtils';
+
+/**
+ * The class that handles the functionality of xrgamepads and controllers
+ */
+export class XRGamepadController {
+    controllers: Array<Controller>;
+    toStreamerMessagesProvider: StreamMessageController;
+
+    /**
+     * @param toStreamerMessagesProvider - Stream message instance
+     */
+    constructor(toStreamerMessagesProvider: StreamMessageController) {
+        this.toStreamerMessagesProvider = toStreamerMessagesProvider;
+        this.controllers = [];
+    }
+
+    updateStatus(
+        source: XRInputSource,
+        frame: XRFrame,
+        refSpace: XRReferenceSpace
+    ) {
+        if (source.gamepad) {
+            const gamepadPose = frame.getPose(source.gripSpace, refSpace);
+            if (!gamepadPose) {
+                return;
+            }
+
+            let system = 0;
+            if (source.profiles.includes('htc-vive')) {
+                system = 1;
+            } else if (source.profiles.includes('oculus-touch')) {
+                system = 2;
+            }
+            // TODO (william.belcher): Add other profiles (Quest, Microsoft Mixed Reality, etc)
+            this.toStreamerMessagesProvider.toStreamerHandlers.get('XRSystem')([
+                system
+            ]);
+
+            // Default: AnyHand (2)
+            let handedness = 2;
+            switch (source.handedness) {
+                case 'left':
+                    handedness = 0;
+                    break;
+                case 'right':
+                    handedness = 1;
+                    break;
+            }
+
+            // Send controller transform
+            const matrix = gamepadPose.transform.matrix;
+            const mat = [];
+            for (let i = 0; i < 16; i++) {
+                mat[i] = new Float32Array([matrix[i]])[0];
+            }
+
+            // prettier-ignore
+            this.toStreamerMessagesProvider.toStreamerHandlers.get('XRControllerTransform')([
+                mat[0], mat[4], mat[8], mat[12],
+                mat[1], mat[5], mat[9], mat[13],
+                mat[2], mat[6], mat[10], mat[14],
+                mat[3], mat[7], mat[11], mat[15],
+                handedness
+            ]);
+
+            // Handle controller buttons and axes
+            if (this.controllers[handedness] === undefined) {
+                this.controllers[handedness] = {
+                    prevState: undefined,
+                    currentState: undefined,
+					id: undefined
+                };
+                this.controllers[handedness].prevState =
+                    WebXRUtils.deepCopyGamepad(source.gamepad);
+            }
+
+            this.controllers[handedness].currentState =
+                WebXRUtils.deepCopyGamepad(source.gamepad);
+
+            const controller = this.controllers[handedness];
+            const currState = controller.currentState;
+            const prevState = controller.prevState;
+            // Iterate over buttons
+            for (let i = 0; i < currState.buttons.length; i++) {
+                const currButton = currState.buttons[i];
+                const prevButton = prevState.buttons[i];
+
+                if (currButton.pressed) {
+                    // press
+                    this.toStreamerMessagesProvider.toStreamerHandlers.get(
+                        'XRButtonPressed'
+                    )([handedness, i, prevButton.pressed ? 1 : 0]);
+                } else if (!currButton.pressed && prevButton.pressed) {
+                    this.toStreamerMessagesProvider.toStreamerHandlers.get(
+                        'XRButtonReleased'
+                    )([handedness, i, 0]);
+                }
+
+                if (currButton.touched && !currButton.pressed) {
+                    // press
+                    this.toStreamerMessagesProvider.toStreamerHandlers.get(
+                        'XRButtonPressed'
+                    )([handedness, 3, prevButton.touched ? 1 : 0]);
+                } else if (!currButton.touched && prevButton.touched) {
+                    this.toStreamerMessagesProvider.toStreamerHandlers.get(
+                        'XRButtonReleased'
+                    )([handedness, 3, 0]);
+                }
+            }
+
+            // Iterate over gamepad axes
+            for (let i = 0; i < currState.axes.length; i++) {
+                this.toStreamerMessagesProvider.toStreamerHandlers.get(
+                    'XRAnalog'
+                )([handedness, i, currState.axes[i]]);
+            }
+
+            this.controllers[handedness].prevState = currState;
+        }
+    }
+}

package/src/PeerConnectionController/AggregatedStats.ts

@@ -0,0 +1,311 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import {
+    InboundRTPStats,
+    InboundVideoStats,
+    InboundAudioStats
+} from './InboundRTPStats';
+import { InboundTrackStats } from './InboundTrackStats';
+import { DataChannelStats } from './DataChannelStats';
+import { CandidateStat } from './CandidateStat';
+import { CandidatePairStats } from './CandidatePairStats';
+import { OutBoundRTPStats, OutBoundVideoStats } from './OutBoundRTPStats';
+import { SessionStats } from './SessionStats';
+import { StreamStats } from './StreamStats';
+import { CodecStats } from './CodecStats';
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+
+/**
+ * The Aggregated Stats that is generated from the RTC Stats Report
+ */
+
+type RTCStatsTypePS = RTCStatsType | 'stream' | 'media-playout';
+export class AggregatedStats {
+    inboundVideoStats: InboundVideoStats;
+    inboundAudioStats: InboundAudioStats;
+    lastVideoStats: InboundVideoStats;
+    lastAudioStats: InboundAudioStats;
+    candidatePair: CandidatePairStats;
+    DataChannelStats: DataChannelStats;
+    localCandidates: Array<CandidateStat>;
+    remoteCandidates: Array<CandidateStat>;
+    outBoundVideoStats: OutBoundVideoStats;
+    sessionStats: SessionStats;
+    streamStats: StreamStats;
+    codecs: Map<string, string>;
+
+    constructor() {
+        this.inboundVideoStats = new InboundVideoStats();
+        this.inboundAudioStats = new InboundAudioStats();
+        this.candidatePair = new CandidatePairStats();
+        this.DataChannelStats = new DataChannelStats();
+        this.outBoundVideoStats = new OutBoundVideoStats();
+        this.sessionStats = new SessionStats();
+        this.streamStats = new StreamStats();
+        this.codecs = new Map<string, string>();
+    }
+
+    /**
+     * Gather all the information from the RTC Peer Connection Report
+     * @param rtcStatsReport - RTC Stats Report
+     */
+    processStats(rtcStatsReport: RTCStatsReport) {
+        this.localCandidates = new Array<CandidateStat>();
+        this.remoteCandidates = new Array<CandidateStat>();
+
+        rtcStatsReport.forEach((stat) => {
+            const type: RTCStatsTypePS = stat.type;
+
+            switch (type) {
+                case 'candidate-pair':
+                    this.handleCandidatePair(stat);
+                    break;
+                case 'certificate':
+                    break;
+                case 'codec':
+                    this.handleCodec(stat);
+                    break;
+                case 'data-channel':
+                    this.handleDataChannel(stat);
+                    break;
+                case 'inbound-rtp':
+                    this.handleInBoundRTP(stat);
+                    break;
+                case 'local-candidate':
+                    this.handleLocalCandidate(stat);
+                    break;
+                case 'media-source':
+                    break;
+                case 'media-playout':
+                    break;
+                case 'outbound-rtp':
+                    break;
+                case 'peer-connection':
+                    break;
+                case 'remote-candidate':
+                    this.handleRemoteCandidate(stat);
+                    break;
+                case 'remote-inbound-rtp':
+                    break;
+                case 'remote-outbound-rtp':
+                    this.handleRemoteOutBound(stat);
+                    break;
+                case 'track':
+                    this.handleTrack(stat);
+                    break;
+                case 'transport':
+                    break;
+                case 'stream':
+                    this.handleStream(stat);
+                    break;
+                default:
+                    Logger.Error(Logger.GetStackTrace(), 'unhandled Stat Type');
+                    Logger.Log(Logger.GetStackTrace(), stat);
+                    break;
+            }
+        });
+    }
+
+    /**
+     * Process stream stats data from webrtc
+     *
+     * @param stat - the stats coming in from webrtc
+     */
+    handleStream(stat: StreamStats) {
+        this.streamStats = stat;
+    }
+
+    /**
+     * Process the Ice Candidate Pair Data
+     * @param stat - the stats coming in from ice candidates
+     */
+    handleCandidatePair(stat: CandidatePairStats) {
+        this.candidatePair.bytesReceived = stat.bytesReceived;
+        this.candidatePair.bytesSent = stat.bytesSent;
+        this.candidatePair.localCandidateId = stat.localCandidateId;
+        this.candidatePair.remoteCandidateId = stat.remoteCandidateId;
+        this.candidatePair.nominated = stat.nominated;
+        this.candidatePair.readable = stat.readable;
+        this.candidatePair.selected = stat.selected;
+        this.candidatePair.writable = stat.writable;
+        this.candidatePair.state = stat.state;
+        this.candidatePair.currentRoundTripTime = stat.currentRoundTripTime;
+    }
+
+    /**
+     * Process the Data Channel Data
+     * @param stat - the stats coming in from the data channel
+     */
+    handleDataChannel(stat: DataChannelStats) {
+        this.DataChannelStats.bytesReceived = stat.bytesReceived;
+        this.DataChannelStats.bytesSent = stat.bytesSent;
+        this.DataChannelStats.dataChannelIdentifier =
+            stat.dataChannelIdentifier;
+        this.DataChannelStats.id = stat.id;
+        this.DataChannelStats.label = stat.label;
+        this.DataChannelStats.messagesReceived = stat.messagesReceived;
+        this.DataChannelStats.messagesSent = stat.messagesSent;
+        this.DataChannelStats.protocol = stat.protocol;
+        this.DataChannelStats.state = stat.state;
+        this.DataChannelStats.timestamp = stat.timestamp;
+    }
+
+    /**
+     * Process the Local Ice Candidate Data
+     * @param stat - local stats
+     */
+    handleLocalCandidate(stat: CandidateStat) {
+        const localCandidate = new CandidateStat();
+        localCandidate.label = 'local-candidate';
+        localCandidate.address = stat.address;
+        localCandidate.port = stat.port;
+        localCandidate.protocol = stat.protocol;
+        localCandidate.candidateType = stat.candidateType;
+        localCandidate.id = stat.id;
+        this.localCandidates.push(localCandidate);
+    }
+
+    /**
+     * Process the Remote Ice Candidate Data
+     * @param stat - ice candidate stats
+     */
+    handleRemoteCandidate(stat: CandidateStat) {
+        const RemoteCandidate = new CandidateStat();
+        RemoteCandidate.label = 'local-candidate';
+        RemoteCandidate.address = stat.address;
+        RemoteCandidate.port = stat.port;
+        RemoteCandidate.protocol = stat.protocol;
+        RemoteCandidate.id = stat.id;
+        RemoteCandidate.candidateType = stat.candidateType;
+        this.remoteCandidates.push(RemoteCandidate);
+    }
+
+    /**
+     * Process the Inbound RTP Audio and Video Data
+     * @param stat - inbound rtp stats
+     */
+    handleInBoundRTP(stat: InboundRTPStats) {
+        switch (stat.kind) {
+            case 'video':
+                // Need to convert to unknown first to remove an error around
+                // InboundVideoStats having the bitrate member which isn't found on
+                // the InboundRTPStats
+                this.inboundVideoStats = stat as unknown as InboundVideoStats;
+
+                if (this.lastVideoStats != undefined) {
+                    this.inboundVideoStats.bitrate =
+                        (8 *
+                            (this.inboundVideoStats.bytesReceived -
+                                this.lastVideoStats.bytesReceived)) /
+                        (this.inboundVideoStats.timestamp -
+                            this.lastVideoStats.timestamp);
+                    this.inboundVideoStats.bitrate = Math.floor(
+                        this.inboundVideoStats.bitrate
+                    );
+                }
+                this.lastVideoStats = { ...this.inboundVideoStats };
+                break;
+            case 'audio':
+                // Need to convert to unknown first to remove an error around
+                // InboundAudioStats having the bitrate member which isn't found on
+                // the InboundRTPStats
+                this.inboundAudioStats = stat as unknown as InboundAudioStats;
+
+                if (this.lastAudioStats != undefined) {
+                    this.inboundAudioStats.bitrate =
+                        (8 *
+                            (this.inboundAudioStats.bytesReceived -
+                                this.lastAudioStats.bytesReceived)) /
+                        (this.inboundAudioStats.timestamp -
+                            this.lastAudioStats.timestamp);
+                    this.inboundAudioStats.bitrate = Math.floor(
+                        this.inboundAudioStats.bitrate
+                    );
+                }
+                this.lastAudioStats = { ...this.inboundAudioStats };
+                break;
+            default:
+                Logger.Log(Logger.GetStackTrace(), 'Kind is not handled');
+                break;
+        }
+    }
+
+    /**
+     * Process the outbound RTP Audio and Video Data
+     * @param stat - remote outbound stats
+     */
+    handleRemoteOutBound(stat: OutBoundRTPStats) {
+        switch (stat.kind) {
+            case 'video':
+                this.outBoundVideoStats.bytesSent = stat.bytesSent;
+                this.outBoundVideoStats.id = stat.id;
+                this.outBoundVideoStats.localId = stat.localId;
+                this.outBoundVideoStats.packetsSent = stat.packetsSent;
+                this.outBoundVideoStats.remoteTimestamp = stat.remoteTimestamp;
+                this.outBoundVideoStats.timestamp = stat.timestamp;
+                break;
+            case 'audio':
+                break;
+
+            default:
+                break;
+        }
+    }
+
+    /**
+     * Process the Inbound Video Track Data
+     * @param stat - video track stats
+     */
+    handleTrack(stat: InboundTrackStats) {
+        // we only want to extract stats from the video track
+        if (
+            stat.type === 'track' &&
+            (stat.trackIdentifier === 'video_label' || stat.kind === 'video')
+        ) {
+            this.inboundVideoStats.framesDropped = stat.framesDropped;
+            this.inboundVideoStats.framesReceived = stat.framesReceived;
+            this.inboundVideoStats.frameHeight = stat.frameHeight;
+            this.inboundVideoStats.frameWidth = stat.frameWidth;
+        }
+    }
+
+    handleCodec(stat: CodecStats) {
+        const codecId = stat.id;
+        const codecType = `${stat.mimeType
+            .replace('video/', '')
+            .replace('audio/', '')}${
+            stat.sdpFmtpLine ? ` ${stat.sdpFmtpLine}` : ''
+        }`;
+        this.codecs.set(codecId, codecType);
+    }
+
+    handleSessionStatistics(
+        videoStartTime: number,
+        inputController: boolean | null,
+        videoEncoderAvgQP: number
+    ) {
+        const deltaTime = Date.now() - videoStartTime;
+        this.sessionStats.runTime = new Date(deltaTime)
+            .toISOString()
+            .substr(11, 8)
+            .toString();
+
+        const controlsStreamInput =
+            inputController === null
+                ? 'Not sent yet'
+                : inputController
+                ? 'true'
+                : 'false';
+        this.sessionStats.controlsStreamInput = controlsStreamInput;
+
+        this.sessionStats.videoEncoderAvgQP = videoEncoderAvgQP;
+    }
+
+    /**
+     * Check if a value coming in from our stats is actually a number
+     * @param value - the number to be checked
+     */
+    isNumber(value: unknown): boolean {
+        return typeof value === 'number' && isFinite(value);
+    }
+}

package/src/PeerConnectionController/CandidatePairStats.ts

@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * ICE Candidate Pair Stats collected from the RTC Stats Report
+ */
+export class CandidatePairStats {
+    bytesReceived: number;
+    bytesSent: number;
+    localCandidateId: string;
+    remoteCandidateId: string;
+    nominated: boolean;
+    readable: boolean;
+    writable: boolean;
+    selected: boolean;
+    state: string;
+    currentRoundTripTime: number;
+}

package/src/PeerConnectionController/CandidateStat.ts

@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * ICE Candidate Stat collected from the RTC Stats Report
+ */
+export class CandidateStat {
+    label: string;
+    id: string;
+    address: string;
+    candidateType: string;
+    port: number;
+    protocol: 'tcp' | 'udp';
+}

package/src/PeerConnectionController/CodecStats.ts

@@ -0,0 +1,19 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Codec Stats collected from the RTC Stats Report
+ */
+export class CodecStats {
+    /* common stats */
+    clockRate: number;
+    id: string;
+    mimeType: string;
+    payloadType: number;
+    sdpFmtpLine: string;
+    timestamp: number;
+    transportId: string;
+    type: string;
+
+    /* audio specific stats */
+    channels: number;
+}

package/src/PeerConnectionController/DataChannelStats.ts

@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Data Channel Stats collected from the RTC Stats Report
+ */
+export class DataChannelStats {
+    bytesReceived: number;
+    bytesSent: number;
+    dataChannelIdentifier: number;
+    id: string;
+    label: string;
+    messagesReceived: number;
+    messagesSent: number;
+    protocol: string;
+    state: string;
+    timestamp: number;
+}

package/src/PeerConnectionController/InboundRTPStats.ts

@@ -0,0 +1,154 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Inbound Audio Stats collected from the RTC Stats Report
+ */
+export class InboundAudioStats {
+    audioLevel: number;
+    bytesReceived: number;
+    codecId: string;
+    concealedSamples: number;
+    concealmentEvents: number;
+    fecPacketsDiscarded: number;
+    fecPacketsReceived: number;
+    headerBytesReceived: number;
+    id: string;
+    insertedSamplesForDeceleration: number;
+    jitter: number;
+    jitterBufferDelay: number;
+    jitterBufferEmittedCount: number;
+    jitterBufferMinimumDelay: number;
+    jitterBufferTargetDelay: number;
+    kind: string;
+    lastPacketReceivedTimestamp: number;
+    mediaType: string;
+    mid: string;
+    packetsDiscarded: number;
+    packetsLost: number;
+    packetsReceived: number;
+    removedSamplesForAcceleration: number;
+    silentConcealedSamples: number;
+    ssrc: number;
+    timestamp: number;
+    totalAudioEnergy: number;
+    totalSamplesDuration: number;
+    totalSamplesReceived: number;
+    trackIdentifier: string;
+    transportId: string;
+    type: string;
+
+    /* additional, custom stats */
+    bitrate: number;
+}
+
+/**
+ * Inbound Video Stats collected from the RTC Stats Report
+ */
+export class InboundVideoStats {
+    bytesReceived: number;
+    codecId: string;
+    firCount: number;
+    frameHeight: number;
+    frameWidth: number;
+    framesAssembledFromMultiplePackets: number;
+    framesDecoded: number;
+    framesDropped: number;
+    framesPerSecond: number;
+    framesReceived: number;
+    freezeCount: number;
+    googTimingFrameInfo: string;
+    headerBytesReceived: number;
+    id: string;
+    jitter: number;
+    jitterBufferDelay: number;
+    jitterBufferEmittedCount: number;
+    keyFramesDecoded: number;
+    kind: string;
+    lastPacketReceivedTimestamp: number;
+    mediaType: string;
+    mid: string;
+    nackCount: number;
+    packetsLost: number;
+    packetsReceived: number;
+    pauseCount: number;
+    pliCount: number;
+    ssrc: number;
+    timestamp: number;
+    totalAssemblyTime: number;
+    totalDecodeTime: number;
+    totalFreezesDuration: number;
+    totalInterFrameDelay: number;
+    totalPausesDuration: number;
+    totalProcessingDelay: number;
+    totalSquaredInterFrameDelay: number;
+    trackIdentifier: string;
+    transportId: string;
+    type: string;
+
+    /* additional, custom stats */
+    bitrate: number;
+}
+
+/**
+ * Inbound Stats collected from the RTC Stats Report
+ */
+export class InboundRTPStats {
+    /* common stats */
+    bytesReceived: number;
+    codecId: string;
+    headerBytesReceived: number;
+    id: string;
+    jitter: number;
+    jitterBufferDelay: number;
+    jitterBufferEmittedCount: number;
+    kind: string;
+    lastPacketReceivedTimestamp: number;
+    mediaType: string;
+    mid: string;
+    packetsLost: number;
+    packetsReceived: number;
+    ssrc: number;
+    timestamp: number;
+    trackIdentifier: string;
+    transportId: string;
+    type: string;
+
+    /* audio specific stats */
+    audioLevel: number;
+    concealedSamples: number;
+    concealmentEvents: number;
+    fecPacketsDiscarded: number;
+    fecPacketsReceived: number;
+    insertedSamplesForDeceleration: number;
+    jitterBufferMinimumDelay: number;
+    jitterBufferTargetDelay: number;
+    packetsDiscarded: number;
+    removedSamplesForAcceleration: number;
+    silentConcealedSamples: number;
+    totalAudioEnergy: number;
+    totalSamplesDuration: number;
+    totalSamplesReceived: number;
+
+    /* video specific stats */
+    firCount: number;
+    frameHeight: number;
+    frameWidth: number;
+    framesAssembledFromMultiplePackets: number;
+    framesDecoded: number;
+    framesDropped: number;
+    framesPerSecond: number;
+    framesReceived: number;
+    freezeCount: number;
+    googTimingFrameInfo: string;
+    keyFramesDecoded: number;
+    nackCount: number;
+    pauseCount: number;
+    pliCount: number;
+    totalAssemblyTime: number;
+    totalDecodeTime: number;
+    totalFreezesDuration: number;
+    totalInterFrameDelay: number;
+    totalPausesDuration: number;
+    totalProcessingDelay: number;
+    totalSquaredInterFrameDelay: number;
+}

package/src/PeerConnectionController/InboundTrackStats.ts

@@ -0,0 +1,34 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Inbound Track Stats collected from the RTC Stats Report
+ */
+export class InboundTrackStats {
+    type: string;
+    kind: string;
+    trackIdentifier: string;
+    receiveToCompositeMs: number;
+    timestamp: number;
+    bytesReceived: number;
+    framesDecoded: number;
+    packetsLost: number;
+    bytesReceivedStart: number;
+    framesDecodedStart: number;
+    timestampStart: number;
+    bitrate: number;
+    lowBitrate: number;
+    highBitrate: number;
+    avgBitrate: number;
+    framerate: number;
+    lowFramerate: number;
+    highFramerate: number;
+    averageFrameRate: number;
+    framesDropped: number;
+    framesReceived: number;
+    framesDroppedPercentage: number;
+    frameHeight: number;
+    frameWidth: number;
+    frameHeightStart: number;
+    frameWidthStart: number;
+    jitter: number;
+}