@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.5

package/src/DataChannel/DataChannelLatencyTestController.ts

@@ -0,0 +1,129 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import {
+    DataChannelLatencyTestRecord,
+    DataChannelLatencyTestRequest,
+    DataChannelLatencyTestResponse,
+    DataChannelLatencyTestResult,
+    DataChannelLatencyTestSeq,
+    DataChannelLatencyTestTimestamp
+} from "./DataChannelLatencyTestResults";
+
+export type DataChannelLatencyTestConfig = {
+    // test duration in milliseconds
+    duration: number;
+    //requests per second
+    rps: number;
+    //request filler size
+    requestSize: number;
+    //response filler size
+    responseSize: number;
+}
+
+export type DataChannelLatencyTestSink = (request: DataChannelLatencyTestRequest) => void;
+export type DataChannelLatencyTestResultCallback = (result: DataChannelLatencyTestResult) => void;
+
+export class DataChannelLatencyTestController {
+    startTime: DataChannelLatencyTestTimestamp;
+    sink: DataChannelLatencyTestSink;
+    callback: DataChannelLatencyTestResultCallback;
+    records: Map<DataChannelLatencyTestSeq, DataChannelLatencyTestRecord>;
+    seq: DataChannelLatencyTestSeq;
+    interval: NodeJS.Timer;
+
+    constructor(sink: DataChannelLatencyTestSink, callback: DataChannelLatencyTestResultCallback) {
+        this.sink = sink;
+        this.callback = callback;
+        this.records = new Map();
+        this.seq = 0;
+    }
+
+    start(config: DataChannelLatencyTestConfig) {
+        if (this.isRunning()) {
+            return false;
+        }
+        this.startTime = Date.now();
+        this.records.clear();
+        this.interval = setInterval((() => {
+            if (Date.now() - this.startTime >= config.duration) {
+                this.stop();
+            } else {
+                this.sendRequest(config.requestSize, config.responseSize);
+            }
+        }).bind(this), Math.floor(1000/config.rps));
+        return true;
+    }
+
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = undefined;
+            this.callback(this.produceResult());
+        }
+    }
+
+    produceResult(): DataChannelLatencyTestResult {
+        const resultRecords = new Map(this.records);
+        return {
+            records: resultRecords,
+            dataChannelRtt: Math.ceil(Array.from(this.records.values()).reduce((acc, next) => {
+                return acc + (next.playerReceivedTimestamp - next.playerSentTimestamp);
+            }, 0) / this.records.size),
+            playerToStreamerTime: Math.ceil(Array.from(this.records.values()).reduce((acc, next) => {
+                return acc + (next.streamerReceivedTimestamp - next.playerSentTimestamp);
+            }, 0) / this.records.size),
+            streamerToPlayerTime: Math.ceil(Array.from(this.records.values()).reduce((acc, next) => {
+                return acc + (next.playerReceivedTimestamp - next.streamerSentTimestamp);
+            }, 0) / this.records.size),
+            exportLatencyAsCSV: () => {
+                let csv = "Timestamp;RTT;PlayerToStreamer;StreamerToPlayer;\n";
+                resultRecords.forEach((record) => {
+                    csv += record.playerSentTimestamp + ";";
+                    csv += (record.playerReceivedTimestamp - record.playerSentTimestamp) + ";";
+                    csv += (record.streamerReceivedTimestamp - record.playerSentTimestamp) + ";";
+                    csv += (record.playerReceivedTimestamp - record.streamerSentTimestamp) + ";";
+                    csv += "\n";
+                })
+                return csv;
+            }
+        }
+    }
+
+    isRunning() {
+        return !!this.interval;
+    }
+
+    receive(response: DataChannelLatencyTestResponse) {
+        if (!this.isRunning()) {
+            return;
+        }
+        if (!response) {
+            Logger.Error(
+                Logger.GetStackTrace(),
+                "Undefined response from server"
+            );
+            return;
+        }
+        let record = this.records.get(response.Seq);
+        if (record) {
+            record.update(response);
+        }
+    }
+
+    sendRequest(requestSize: number, responseSize: number) {
+        let request = this.createRequest(requestSize, responseSize);
+        let record = new DataChannelLatencyTestRecord(request);
+        this.records.set(record.seq, record);
+        this.sink(request);
+    }
+
+    createRequest(requestSize: number, responseSize: number): DataChannelLatencyTestRequest {
+        return {
+            Seq: this.seq++,
+            FillResponseSize: responseSize,
+            Filler: requestSize ? "A".repeat(requestSize) : ""
+        }
+    }
+
+}

package/src/DataChannel/DataChannelLatencyTestResults.ts

@@ -0,0 +1,67 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Data Channel Latency Test types
+ */
+
+
+/**
+ * Unix epoch
+ */
+export type DataChannelLatencyTestTimestamp = number;
+
+/**
+ * Sequence number represented by unsigned int
+ */
+export type DataChannelLatencyTestSeq = number;
+
+/**
+ * Request sent to Streamer
+ */
+export type DataChannelLatencyTestRequest = {
+    Seq: DataChannelLatencyTestSeq;
+    FillResponseSize: number;
+    Filler: string;
+}
+
+/**
+ * Response from the Streamer
+ */
+export type DataChannelLatencyTestResponse = {
+    Seq: DataChannelLatencyTestSeq;
+    Filler: string;
+    ReceivedTimestamp: DataChannelLatencyTestTimestamp;
+    SentTimestamp: DataChannelLatencyTestTimestamp;
+}
+
+export type DataChannelLatencyTestResult = {
+    records: Map<DataChannelLatencyTestSeq, DataChannelLatencyTestRecord>
+    dataChannelRtt: number,
+    playerToStreamerTime: number,
+    streamerToPlayerTime: number,
+    exportLatencyAsCSV: () => string
+}
+
+export class DataChannelLatencyTestRecord {
+    seq: DataChannelLatencyTestSeq;
+    playerSentTimestamp: DataChannelLatencyTestTimestamp;
+    playerReceivedTimestamp: DataChannelLatencyTestTimestamp;
+    streamerReceivedTimestamp: DataChannelLatencyTestTimestamp;
+    streamerSentTimestamp: DataChannelLatencyTestTimestamp;
+    requestFillerSize: number;
+    responseFillerSize: number;
+
+    constructor(request: DataChannelLatencyTestRequest) {
+        this.seq = request.Seq;
+        this.playerSentTimestamp = Date.now();
+        this.requestFillerSize = request.Filler ? request.Filler.length : 0;
+    }
+
+    update(response: DataChannelLatencyTestResponse) {
+        this.playerReceivedTimestamp = Date.now();
+        this.streamerReceivedTimestamp = response.ReceivedTimestamp;
+        this.streamerSentTimestamp = response.SentTimestamp;
+        this.responseFillerSize = response.Filler ? response.Filler.length : 0;
+    }
+
+}

package/src/DataChannel/DataChannelSender.ts

@@ -0,0 +1,59 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { DataChannelController } from './DataChannelController';
+
+/**
+ * A class for sending data channel messages
+ */
+export class DataChannelSender {
+    dataChannelProvider: DataChannelController;
+
+    /**
+     * @param dataChannelProvider - Data channel object type
+     */
+    constructor(dataChannelProvider: DataChannelController) {
+        this.dataChannelProvider = dataChannelProvider;
+    }
+
+    canSend(): boolean {
+        return (
+            this.dataChannelProvider.getDataChannelInstance().dataChannel !==
+                undefined &&
+            this.dataChannelProvider.getDataChannelInstance().dataChannel
+                .readyState == 'open'
+        );
+    }
+
+    /**
+     * Send Data over the Data channel to the UE Instance
+     * @param data - Message Data Array Buffer
+     */
+    sendData(data: ArrayBuffer) {
+        // reset the afk inactivity
+        const dataChannelInstance =
+            this.dataChannelProvider.getDataChannelInstance();
+
+        if (dataChannelInstance.dataChannel.readyState == 'open') {
+            dataChannelInstance.dataChannel.send(data);
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `Message Sent: ${new Uint8Array(data)}`,
+                6
+            );
+            this.resetAfkWarningTimerOnDataSend();
+        } else {
+            Logger.Error(
+                Logger.GetStackTrace(),
+                `Message Failed: ${new Uint8Array(data)}`
+            );
+        }
+    }
+
+    /**
+     * An override method for resetting the Afk warning timer when data is sent over the data channel
+     */
+    resetAfkWarningTimerOnDataSend() {
+        // Base Functionality: Do Nothing
+    }
+}

package/src/DataChannel/InitialSettings.ts

@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Latency Test Results Data
+ */
+export class InitialSettings {
+    PixelStreamingSettings: PixelStreamingSettings;
+    EncoderSettings: EncoderSettings;
+    WebRTCSettings: WebRTCSettings;
+
+    constructor() {
+        this.PixelStreamingSettings = new PixelStreamingSettings();
+        this.EncoderSettings = new EncoderSettings();
+        this.WebRTCSettings = new WebRTCSettings();
+    }
+
+    /**
+     * Checks for compatibility with the FPS and MaxFPS stats between 4.27 and 5
+     */
+    ueCompatible() {
+        if (this.WebRTCSettings.MaxFPS != null) {
+            this.WebRTCSettings.FPS = this.WebRTCSettings.MaxFPS;
+        }
+    }
+}
+
+/**
+ * A class for handling Pixel Streaming details
+ */
+export class PixelStreamingSettings {
+    AllowPixelStreamingCommands?: boolean;
+    DisableLatencyTest?: boolean;
+}
+
+/**
+ * A class for handling encoder stats
+ */
+export class EncoderSettings {
+    TargetBitrate?: number;
+    MaxBitrate?: number;
+    MinQP?: number;
+    MaxQP?: number;
+    RateControl?: 'CBR' | 'VBR' | 'ConstQP';
+    FillerData?: boolean;
+    MultiPass?: 'DISABLED' | 'QUARTER' | 'FULL';
+}
+
+/**
+ * A class for handling web rtc stats
+ */
+export class WebRTCSettings {
+    DegradationPref?: 'BALANCED' | 'MAINTAIN_FRAMERATE' | 'MAINTAIN_RESOLUTION';
+    MinBitrate?: number;
+    MaxBitrate?: number;
+    LowQP?: number;
+    HighQP?: number;
+    // UE4.27 compatible
+    MaxFPS?: number;
+    // UE5 compatible
+    FPS?: number;
+}

package/src/DataChannel/LatencyTestResults.ts

@@ -0,0 +1,76 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+/**
+ * Latency Test Results Data
+ */
+export class LatencyTestResults {
+    //Fields Set from the latency payload regardless of version
+    ReceiptTimeMs: number = null;
+    TransmissionTimeMs: number = null;
+
+    //Fields Set from the latency payload from 4.27.2
+    PreCaptureTimeMs: number = null;
+    PostCaptureTimeMs: number = null;
+    PreEncodeTimeMs: number = null;
+    PostEncodeTimeMs: number = null;
+
+    //Fields Set from the latency payload from 5.0
+    EncodeMs: number = null;
+    CaptureToSendMs: number = null;
+
+    //Fields Set when processed
+    testStartTimeMs = 0;
+    browserReceiptTimeMs = 0;
+
+    //Fields set from calculations
+    latencyExcludingDecode = 0;
+    testDuration = 0;
+    //ueLatency: number = 0;
+    networkLatency = 0;
+    browserSendLatency = 0;
+    frameDisplayDeltaTimeMs = 0;
+    endToEndLatency = 0;
+    //uePixelStreamLatency: number = 0;
+    encodeLatency = 0;
+
+    /**
+     * Sets the Delta Time Milliseconds
+     * @param DeltaTimeMs - Delta Time Milliseconds
+     */
+    setFrameDisplayDeltaTime(DeltaTimeMs: number) {
+        if (this.frameDisplayDeltaTimeMs == 0) {
+            this.frameDisplayDeltaTimeMs = Math.round(DeltaTimeMs);
+        }
+    }
+
+    /**
+     * Process the encoder times and set them
+     */
+    processFields() {
+        if (
+            this.EncodeMs == null &&
+            (this.PreEncodeTimeMs != null || this.PostEncodeTimeMs != null)
+        ) {
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `Setting Encode Ms \n ${this.PostEncodeTimeMs} \n ${this.PreEncodeTimeMs}`,
+                6
+            );
+            this.EncodeMs = this.PostEncodeTimeMs - this.PreEncodeTimeMs;
+        }
+
+        if (
+            this.CaptureToSendMs == null &&
+            (this.PreCaptureTimeMs != null || this.PostCaptureTimeMs != null)
+        ) {
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `Setting CaptureToSendMs Ms \n ${this.PostCaptureTimeMs} \n ${this.PreCaptureTimeMs}`,
+                6
+            );
+            this.CaptureToSendMs =
+                this.PostCaptureTimeMs - this.PreCaptureTimeMs;
+        }
+    }
+}

package/src/FreezeFrame/FreezeFrame.ts

@@ -0,0 +1,114 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * A class for managing the freeze frame object
+ */
+export class FreezeFrame {
+    protected rootDiv: HTMLElement;
+    protected rootElement: HTMLDivElement;
+    imageElement: HTMLImageElement;
+    freezeFrameHeight = 0;
+    freezeFrameWidth = 0;
+
+    /**
+     * Construct a freeze frame
+     * @param rootDiv the div that a freeze frame element will be injected into
+     */
+    constructor(rootDiv: HTMLElement) {
+        this.rootDiv = rootDiv;
+
+        // create the overlay
+        this.rootElement = document.createElement('div');
+        this.rootElement.id = 'freezeFrame';
+        this.rootElement.style.display = 'none';
+        this.rootElement.style.pointerEvents = 'none';
+        this.rootElement.style.position = 'absolute';
+        this.rootElement.style.zIndex = '20';
+
+        // create the image place holder
+        this.imageElement = document.createElement('img');
+        this.imageElement.style.position = 'absolute';
+
+        // append the image into the root element and append the element to the root div
+        this.rootElement.appendChild(this.imageElement);
+        this.rootDiv.appendChild(this.rootElement);
+    }
+
+    /**
+     * Set the freeze frame element for showing
+     */
+    setElementForShow() {
+        this.rootElement.style.display = 'block';
+    }
+
+    /**
+     * Set the freeze frame element for hiding
+     */
+    setElementForHide() {
+        this.rootElement.style.display = 'none';
+    }
+
+    /**
+     * Update the freeze frames image source
+     * @param jpeg - the freeze frame image as a byte array data
+     */
+    updateImageElementSource(jpeg: Uint8Array) {
+        const base64 = btoa(
+            jpeg.reduce((data, byte) => data + String.fromCharCode(byte), '')
+        );
+        this.imageElement.src = 'data:image/jpeg;base64,' + base64;
+    }
+
+    /**
+     * Set the dimensions for the freeze frame from the element and resize it
+     */
+    setDimensionsFromElementAndResize() {
+        this.freezeFrameHeight = this.imageElement.naturalHeight;
+        this.freezeFrameWidth = this.imageElement.naturalWidth;
+        this.resize();
+    }
+
+    /**
+     * Resize a freeze frame element
+     */
+    resize() {
+        if (this.freezeFrameWidth !== 0 && this.freezeFrameHeight !== 0) {
+            let displayWidth = 0;
+            let displayHeight = 0;
+            let displayTop = 0;
+            let displayLeft = 0;
+            const parentAspectRatio =
+                this.rootDiv.clientWidth / this.rootDiv.clientHeight;
+            const videoAspectRatio =
+                this.freezeFrameWidth / this.freezeFrameHeight;
+            if (parentAspectRatio < videoAspectRatio) {
+                displayWidth = this.rootDiv.clientWidth;
+                displayHeight = Math.floor(
+                    this.rootDiv.clientWidth / videoAspectRatio
+                );
+                displayTop = Math.floor(
+                    (this.rootDiv.clientHeight - displayHeight) * 0.5
+                );
+                displayLeft = 0;
+            } else {
+                displayWidth = Math.floor(
+                    this.rootDiv.clientHeight * videoAspectRatio
+                );
+                displayHeight = this.rootDiv.clientHeight;
+                displayTop = 0;
+                displayLeft = Math.floor(
+                    (this.rootDiv.clientWidth - displayWidth) * 0.5
+                );
+            }
+            this.rootElement.style.width = this.rootDiv.offsetWidth + 'px';
+            this.rootElement.style.height = this.rootDiv.offsetHeight + 'px';
+            this.rootElement.style.left = 0 + 'px';
+            this.rootElement.style.top = 0 + 'px';
+
+            this.imageElement.style.width = displayWidth + 'px';
+            this.imageElement.style.height = displayHeight + 'px';
+            this.imageElement.style.left = displayLeft + 'px';
+            this.imageElement.style.top = displayTop + 'px';
+        }
+    }
+}

package/src/FreezeFrame/FreezeFrameController.ts

@@ -0,0 +1,114 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { FreezeFrame } from './FreezeFrame';
+
+/**
+ * A class for controlling freeze frame functionality
+ */
+export class FreezeFrameController {
+    freezeFrame: FreezeFrame;
+    receiving = false;
+    size = 0;
+    jpeg: Uint8Array = undefined;
+    valid = false;
+    freezeFrameDelay = 50;
+
+    /**
+     * Construct a freeze frame controller
+     * @param rootDiv - the div that a freeze frame element will be injected into
+     */
+    constructor(rootDiv: HTMLElement) {
+        this.freezeFrame = new FreezeFrame(rootDiv);
+    }
+
+    /**
+     * Show the freeze frame if it is valid
+     */
+    showFreezeFrame() {
+        if (this.valid) {
+            this.freezeFrame.setElementForShow();
+        }
+    }
+
+    /**
+     * Hide the freeze frame and set the validity to false
+     */
+    hideFreezeFrame() {
+        this.valid = false;
+        this.freezeFrame.setElementForHide();
+    }
+
+    /**
+     * Update the freeze frames image source and load it
+     * @param jpeg - the freeze frame image as a byte array data
+     * @param onLoadCallBack - a call back for managing if the play overlay needs to be shown or not
+     */
+    updateFreezeFrameAndShow(jpeg: Uint8Array, onLoadCallBack: () => void) {
+        this.freezeFrame.updateImageElementSource(jpeg);
+        this.freezeFrame.imageElement.onload = () => {
+            this.freezeFrame.setDimensionsFromElementAndResize();
+            onLoadCallBack();
+        };
+    }
+
+    /**
+     * Process the new freeze frame image and update it
+     * @param view - the freeze frame image as a byte array data
+     * @param onLoadCallBack - a call back for managing if the play overlay needs to be shown or not
+     */
+    processFreezeFrameMessage(view: Uint8Array, onLoadCallBack: () => void) {
+        // Reset freeze frame if we got a freeze frame message and we are not "receiving" yet.
+        if (!this.receiving) {
+            this.receiving = true;
+            this.valid = false;
+            this.size = 0;
+            this.jpeg = undefined;
+        }
+
+        // Extract total size of freeze frame (across all chunks)
+        this.size = new DataView(view.slice(1, 5).buffer).getInt32(0, true);
+
+        // Get the jpeg part of the payload
+        const jpegBytes = view.slice(1 + 4);
+
+        // Append to existing jpeg that holds the freeze frame
+        if (this.jpeg) {
+            const jpeg = new Uint8Array(this.jpeg.length + jpegBytes.length);
+            jpeg.set(this.jpeg, 0);
+            jpeg.set(jpegBytes, this.jpeg.length);
+            this.jpeg = jpeg;
+        }
+        // No existing freeze frame jpeg, make one
+        else {
+            this.jpeg = jpegBytes;
+            this.receiving = true;
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `received first chunk of freeze frame: ${this.jpeg.length}/${this.size}`,
+                6
+            );
+        }
+
+        // Finished receiving freeze frame, we can show it now
+        if (this.jpeg.length === this.size) {
+            this.receiving = false;
+            this.valid = true;
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `received complete freeze frame ${this.size}`,
+                6
+            );
+            this.updateFreezeFrameAndShow(this.jpeg, onLoadCallBack);
+        }
+        // We received more data than the freeze frame payload message indicate (this is an error)
+        else if (this.jpeg.length > this.size) {
+            Logger.Error(
+                Logger.GetStackTrace(),
+                `received bigger freeze frame than advertised: ${this.jpeg.length}/${this.size}`
+            );
+            this.jpeg = undefined;
+            this.receiving = false;
+        }
+    }
+}