@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.5

package/src/UI/OnScreenKeyboard.ts

@@ -0,0 +1,97 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { UnquantizedDenormalizedUnsignedCoord } from '../Util/CoordinateConverter';
+import { MessageReceive } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+
+/**
+ * Class for handling on screen keyboard usage
+ */
+export class OnScreenKeyboard {
+    // If the user focuses on a UE input widget then we show them a button to open
+    // the on-screen keyboard. JavaScript security means we can only show the
+    // on-screen keyboard in response to a user interaction.
+    editTextButton: HTMLButtonElement;
+
+    // A hidden input text box which is used only for focusing and opening the
+    // on-screen keyboard.
+    hiddenInput: HTMLInputElement;
+
+    /**
+     *
+     * @param videoElementParent The div element the video player is injected into
+     */
+    constructor(videoElementParent: HTMLElement) {
+        this.editTextButton = null;
+        this.hiddenInput = null;
+
+        if ('ontouchstart' in document.documentElement) {
+            this.createOnScreenKeyboardHelpers(videoElementParent);
+        }
+    }
+
+    /**
+     * An override for unquantizeAndDenormalizeUnsigned
+     * @param x the x axis point
+     * @param y the y axis point
+     * @returns unquantizeAndDenormalizeUnsigned object
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    unquantizeAndDenormalizeUnsigned(
+        x: number,
+        y: number
+    ): UnquantizedDenormalizedUnsignedCoord {
+        return null;
+    }
+
+    /**
+     * Creates on screen keyboard helpers
+     * @param videoElementParent The div element the video player i injected into
+     */
+    createOnScreenKeyboardHelpers(videoElementParent: HTMLElement) {
+        if (!this.hiddenInput) {
+            this.hiddenInput = document.createElement('input');
+            this.hiddenInput.id = 'hiddenInput';
+            this.hiddenInput.maxLength = 0;
+            videoElementParent.appendChild(this.hiddenInput);
+        }
+
+        if (!this.editTextButton) {
+            this.editTextButton = document.createElement('button');
+            this.editTextButton.id = 'editTextButton';
+            this.editTextButton.innerHTML = 'edit text';
+            videoElementParent.appendChild(this.editTextButton);
+
+            // Hide the 'edit text' button.
+            this.editTextButton.classList.add('hiddenState');
+
+            this.editTextButton.addEventListener('touchend', (event: Event) => {
+                // Show the on-screen keyboard.
+                this.hiddenInput.focus();
+                event.preventDefault();
+            });
+        }
+    }
+
+    /**
+     * Shows the on screen keyboard
+     * @param command the command received via the data channel containing keyboard positions
+     */
+    showOnScreenKeyboard(command: MessageReceive.MessageOnScreenKeyboard) {
+        if (command.showOnScreenKeyboard) {
+            // Show the 'edit text' button.
+            this.editTextButton.classList.remove('hiddenState');
+            // Place the 'edit text' button near the UE input widget.
+            const pos = this.unquantizeAndDenormalizeUnsigned(
+                command.x,
+                command.y
+            );
+            this.editTextButton.style.top = pos.y.toString() + 'px';
+            this.editTextButton.style.left = (pos.x - 40).toString() + 'px';
+        } else {
+            // Hide the 'edit text' button.
+            this.editTextButton.classList.add('hiddenState');
+            // Hide the on-screen keyboard.
+            this.hiddenInput.blur();
+        }
+    }
+}

package/src/UeInstanceMessage/ResponseController.ts

@@ -0,0 +1,47 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+
+export class ResponseController {
+    responseEventListeners: Map<string, (response: string) => void> = new Map();
+
+    /**
+     * Add a response event listener to the response map
+     * @param name - The name of the response
+     * @param listener - The method to be activated when the response is selected
+     */
+    addResponseEventListener(
+        name: string,
+        listener: (response: string) => void
+    ) {
+        this.responseEventListeners.set(name, listener);
+    }
+
+    /**
+     * Remove a response event listener to the response map
+     * @param name - The name of the response
+     */
+    removeResponseEventListener(name: string) {
+        this.responseEventListeners.delete(name);
+    }
+
+    /**
+     * Handle a response when receiving one form the streamer
+     * @param message - Data received from the data channel with the command in question
+     */
+    onResponse(message: ArrayBuffer) {
+        Logger.Log(
+            Logger.GetStackTrace(),
+            'DataChannelReceiveMessageType.Response',
+            6
+        );
+        const responses = new TextDecoder('utf-16').decode(message.slice(1));
+
+        Logger.Log(Logger.GetStackTrace(), responses, 6);
+        this.responseEventListeners.forEach(
+            (listener: (response: string) => void) => {
+                listener(responses);
+            }
+        );
+    }
+}

package/src/UeInstanceMessage/SendMessageController.ts

@@ -0,0 +1,154 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { DataChannelSender } from '../DataChannel/DataChannelSender';
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { StreamMessageController } from './StreamMessageController';
+
+export class SendMessageController {
+    toStreamerMessagesMapProvider: StreamMessageController;
+    dataChannelSender: DataChannelSender;
+
+    /**
+     * @param dataChannelSender - Data channel instance
+     * @param toStreamerMessagesMapProvider - Stream Messages instance
+     */
+    constructor(
+        dataChannelSender: DataChannelSender,
+        toStreamerMessagesMapProvider: StreamMessageController
+    ) {
+        this.dataChannelSender = dataChannelSender;
+        this.toStreamerMessagesMapProvider = toStreamerMessagesMapProvider;
+    }
+
+    /**
+     * Send a message to the streamer through the data channel
+     * @param messageType - the type of message we are sending
+     * @param messageData - the message data we are sending over the data channel
+     * @returns - nil
+     */
+    sendMessageToStreamer(messageType: string, messageData?: Array<number | string>) {
+        if (messageData === undefined) {
+            messageData = [];
+        }
+
+        const toStreamerMessages =
+            this.toStreamerMessagesMapProvider.toStreamerMessages;
+        const messageFormat = toStreamerMessages.get(messageType);
+        if (messageFormat === undefined) {
+            Logger.Error(
+                Logger.GetStackTrace(),
+                `Attempted to send a message to the streamer with message type: ${messageType}, but the frontend hasn't been configured to send such a message. Check you've added the message type in your cpp`
+            );
+            return;
+        }
+
+        if(messageFormat.structure && messageData && messageFormat.structure.length !== messageData.length) {
+            Logger.Error(
+                Logger.GetStackTrace(),
+                `Provided message data doesn't match expected layout. Expected [ ${messageFormat.structure.map((element: string) => {
+                    switch (element) {
+                        case 'uint8':
+                        case 'uint16':
+                        case 'int16':
+                        case 'float':
+                        case 'double':
+                            return 'number';
+                        case 'string':
+                            return 'string';
+                    }
+                }).toString() } ] but received [ ${messageData.map((element: number | string) => typeof element).toString()} ]`
+            );
+            return;
+        }
+
+        let byteLength = 0;
+        const textEncoder = new TextEncoder();
+        // One loop to calculate the length in bytes of all of the provided data
+        messageData.forEach((element: number | string, idx: number) => {
+            const type = messageFormat.structure[idx];
+            switch (type) {
+                case 'uint8':
+                    byteLength += 1;
+                    break;
+
+                case 'uint16':
+                    byteLength += 2;
+                    break;
+
+                case 'int16':
+                    byteLength += 2;
+                    break;
+
+                case 'float':
+                    byteLength += 4;
+                    break;
+
+                case 'double':
+                    byteLength += 8;
+                    break;
+
+                case 'string':
+                    // 2 bytes for string length
+                    byteLength += 2;
+                    // 2 bytes per characters
+                    byteLength += 2 * textEncoder.encode(element as string).length;
+                    break;
+            }
+        });
+
+        const data = new DataView(new ArrayBuffer(byteLength + 1));
+        data.setUint8(0, messageFormat.id);
+        let byteOffset = 1;
+
+        messageData.forEach((element: number | string, idx: number) => {
+            const type = messageFormat.structure[idx];
+            switch (type) {
+                case 'uint8':
+                    data.setUint8(byteOffset, element as number);
+                    byteOffset += 1;
+                    break;
+
+                case 'uint16':
+                    data.setUint16(byteOffset, element as number, true);
+                    byteOffset += 2;
+                    break;
+
+                case 'int16':
+                    data.setInt16(byteOffset, element as number, true);
+                    byteOffset += 2;
+                    break;
+
+                case 'float':
+                    data.setFloat32(byteOffset, element as number, true);
+                    byteOffset += 4;
+                    break;
+
+                case 'double':
+                    data.setFloat64(byteOffset, element as number, true);
+                    byteOffset += 8;
+                    break;
+
+                case 'string':
+                    data.setUint16(byteOffset, (element as string).length, true);
+                    byteOffset += 2;
+                    for (let i = 0; i < (element as string).length; i++) {
+                        data.setUint16(byteOffset, (element as string).charCodeAt(i), true);
+                        byteOffset += 2;
+                    }
+                    break;
+            }
+        });
+
+        if (!this.dataChannelSender.canSend()) {
+            Logger.Info(
+                Logger.GetStackTrace(),
+                `Data channel cannot send yet, skipping sending message: ${messageType} - ${new Uint8Array(
+                    data.buffer
+                )}`
+            );
+            return;
+        }
+
+        this.dataChannelSender.sendData(data.buffer);
+    }
+}

package/src/UeInstanceMessage/StreamMessageController.ts

@@ -0,0 +1,233 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+
+export class ToStreamerMessage {
+    id: number;
+    structure?: Array<string>;
+}
+
+export class StreamMessageController {
+    toStreamerHandlers: Map<
+        string,
+        (messageData?: Array<number | string> | undefined) => void
+    >;
+    fromStreamerHandlers: Map<
+        string,
+        (messageType: string, messageData?: ArrayBuffer | undefined) => void
+    >;
+    //                        Type      Format
+    toStreamerMessages: Map<string, ToStreamerMessage>;
+    //                         ID      Type
+    fromStreamerMessages: Map<number, string>;
+
+    constructor() {
+        this.toStreamerHandlers = new Map();
+        this.fromStreamerHandlers = new Map();
+        this.toStreamerMessages = new Map();
+        this.fromStreamerMessages = new Map();
+    }
+
+    /**
+     * Populate the Default message protocol
+     */
+    populateDefaultProtocol() {
+        /*
+         * Control Messages. Range = 0..49.
+         */
+        this.toStreamerMessages.set('IFrameRequest', {
+            id: 0,
+            structure: []
+        });
+        this.toStreamerMessages.set('RequestQualityControl', {
+            id: 1,
+            structure: []
+        });
+        this.toStreamerMessages.set('FpsRequest', {
+            id: 2,
+            structure: []
+        });
+        this.toStreamerMessages.set('AverageBitrateRequest', {
+            id: 3,
+            structure: []
+        });
+        this.toStreamerMessages.set('StartStreaming', {
+            id: 4,
+            structure: []
+        });
+        this.toStreamerMessages.set('StopStreaming', {
+            id: 5,
+            structure: []
+        });
+        this.toStreamerMessages.set('LatencyTest', {
+            id: 6,
+            structure: ['string']
+        });
+        this.toStreamerMessages.set('RequestInitialSettings', {
+            id: 7,
+            structure: []
+        });
+        this.toStreamerMessages.set('TestEcho', {
+            id: 8,
+            structure: []
+        });
+        this.toStreamerMessages.set('DataChannelLatencyTest', {
+            id: 9,
+            structure: []
+        });
+        /*
+         * Input Messages. Range = 50..89.
+         */
+        // Generic Input Messages. Range = 50..59.
+        this.toStreamerMessages.set('UIInteraction', {
+            id: 50,
+            structure: ['string']
+        });
+        this.toStreamerMessages.set('Command', {
+            id: 51,
+            structure: ['string']
+        });
+        // Keyboard Input Message. Range = 60..69.
+        this.toStreamerMessages.set('KeyDown', {
+            id: 60,
+            //            keyCode  isRepeat
+            structure: ['uint8', 'uint8']
+        });
+        this.toStreamerMessages.set('KeyUp', {
+            id: 61,
+            //            keyCode
+            structure: ['uint8']
+        });
+        this.toStreamerMessages.set('KeyPress', {
+            id: 62,
+            //            charcode
+            structure: ['uint16']
+        });
+        // Mouse Input Messages. Range = 70..79.
+        this.toStreamerMessages.set('MouseEnter', {
+            id: 70,
+            structure: []
+        });
+        this.toStreamerMessages.set('MouseLeave', {
+            id: 71,
+            structure: []
+        });
+        this.toStreamerMessages.set('MouseDown', {
+            id: 72,
+            //              button     x         y
+            structure: ['uint8', 'uint16', 'uint16']
+        });
+        this.toStreamerMessages.set('MouseUp', {
+            id: 73,
+            //              button     x         y
+            structure: ['uint8', 'uint16', 'uint16']
+        });
+        this.toStreamerMessages.set('MouseMove', {
+            id: 74,
+            //              x           y      deltaX    deltaY
+            structure: ['uint16', 'uint16', 'int16', 'int16']
+        });
+        this.toStreamerMessages.set('MouseWheel', {
+            id: 75,
+            //              delta       x        y
+            structure: ['int16', 'uint16', 'uint16']
+        });
+        this.toStreamerMessages.set('MouseDouble', {
+            id: 76,
+            //              button     x         y
+            structure: ['uint8', 'uint16', 'uint16']
+        });
+        // Touch Input Messages. Range = 80..89.
+        this.toStreamerMessages.set('TouchStart', {
+            id: 80,
+            //          numtouches(1)   x       y        idx     force     valid
+            structure: ['uint8', 'uint16', 'uint16', 'uint8', 'uint8', 'uint8']
+        });
+        this.toStreamerMessages.set('TouchEnd', {
+            id: 81,
+            //          numtouches(1)   x       y        idx     force     valid
+            structure: ['uint8', 'uint16', 'uint16', 'uint8', 'uint8', 'uint8']
+        });
+        this.toStreamerMessages.set('TouchMove', {
+            id: 82,
+            //          numtouches(1)   x       y       idx      force     valid
+            structure: ['uint8', 'uint16', 'uint16', 'uint8', 'uint8', 'uint8']
+        });
+        // Gamepad Input Messages. Range = 90..99
+        this.toStreamerMessages.set('GamepadConnected', {
+            id: 93,
+            structure: []
+        });
+        this.toStreamerMessages.set('GamepadButtonPressed', {
+            id: 90,
+            //         ctrlerId   button  isRepeat
+            structure: ['uint8', 'uint8', 'uint8']
+        });
+        this.toStreamerMessages.set('GamepadButtonReleased', {
+            id: 91,
+            //         ctrlerId   button  isRepeat(0)
+            structure: ['uint8', 'uint8', 'uint8']
+        });
+        this.toStreamerMessages.set('GamepadAnalog', {
+            id: 92,
+            //         ctrlerId   button  analogValue
+            structure: ['uint8', 'uint8', 'double']
+        });
+        this.toStreamerMessages.set('GamepadDisconnected', {
+            id: 94,
+            //          ctrlerId
+            structure: ['uint8']
+        });
+
+        this.fromStreamerMessages.set(0, 'QualityControlOwnership');
+        this.fromStreamerMessages.set(1, 'Response');
+        this.fromStreamerMessages.set(2, 'Command');
+        this.fromStreamerMessages.set(3, 'FreezeFrame');
+        this.fromStreamerMessages.set(4, 'UnfreezeFrame');
+        this.fromStreamerMessages.set(5, 'VideoEncoderAvgQP');
+        this.fromStreamerMessages.set(6, 'LatencyTest');
+        this.fromStreamerMessages.set(7, 'InitialSettings');
+        this.fromStreamerMessages.set(8, 'FileExtension');
+        this.fromStreamerMessages.set(9, 'FileMimeType');
+        this.fromStreamerMessages.set(10, 'FileContents');
+        this.fromStreamerMessages.set(11, 'TestEcho');
+        this.fromStreamerMessages.set(12, 'InputControlOwnership');
+        this.fromStreamerMessages.set(13, 'GamepadResponse');
+        this.fromStreamerMessages.set(14, 'DataChannelLatencyTest');
+        this.fromStreamerMessages.set(255, 'Protocol');
+    }
+
+    /**
+     * Register a message handler
+     * @param messageDirection - the direction of the message; toStreamer or fromStreamer
+     * @param messageType - the type of the message
+     * @param messageHandler - the function or method to be executed when this handler is called
+     */
+    registerMessageHandler(
+        messageDirection: MessageDirection,
+        messageType: string,
+        messageHandler: (messageData?: unknown | undefined) => void
+    ) {
+        switch (messageDirection) {
+            case MessageDirection.ToStreamer:
+                this.toStreamerHandlers.set(messageType, messageHandler);
+                break;
+            case MessageDirection.FromStreamer:
+                this.fromStreamerHandlers.set(messageType, messageHandler);
+                break;
+            default:
+                Logger.Log(
+                    Logger.GetStackTrace(),
+                    `Unknown message direction ${messageDirection}`
+                );
+        }
+    }
+}
+
+/**
+ * The enum for message directions
+ */
+export enum MessageDirection {
+    ToStreamer = 0,
+    FromStreamer = 1
+}

package/src/UeInstanceMessage/ToStreamerMessagesController.ts

@@ -0,0 +1,62 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { SendMessageController } from './SendMessageController';
+
+export class ToStreamerMessagesController {
+    sendMessageController: SendMessageController;
+
+    /**
+     * @param sendMessageController - Stream message controller instance
+     */
+    constructor(sendMessageController: SendMessageController) {
+        this.sendMessageController = sendMessageController;
+    }
+
+    /**
+     * Send Request to Take Quality Control to the UE Instance
+     */
+    SendRequestQualityControl() {
+        this.sendMessageController.sendMessageToStreamer(
+            'RequestQualityControl'
+        );
+    }
+
+    /**
+     * Send Max FPS Request to the UE Instance
+     */
+    SendMaxFpsRequest() {
+        this.sendMessageController.sendMessageToStreamer('FpsRequest');
+    }
+
+    /**
+     * Send Average Bitrate Request to the UE Instance
+     */
+    SendAverageBitrateRequest() {
+        this.sendMessageController.sendMessageToStreamer(
+            'AverageBitrateRequest'
+        );
+    }
+
+    /**
+     * Send a Start Streaming Message to the UE Instance
+     */
+    SendStartStreaming() {
+        this.sendMessageController.sendMessageToStreamer('StartStreaming');
+    }
+
+    /**
+     * Send a Stop Streaming Message to the UE Instance
+     */
+    SendStopStreaming() {
+        this.sendMessageController.sendMessageToStreamer('StopStreaming');
+    }
+
+    /**
+     * Send a Request Initial Settings to the UE Instance
+     */
+    SendRequestInitialSettings() {
+        this.sendMessageController.sendMessageToStreamer(
+            'RequestInitialSettings'
+        );
+    }
+}