@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.5

package/src/Inputs/MouseController.ts

@@ -0,0 +1,362 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { MouseButtonsMask, MouseButton } from './MouseButtons';
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController';
+import { CoordinateConverter } from '../Util/CoordinateConverter';
+import { VideoPlayer } from '../VideoPlayer/VideoPlayer';
+import { IMouseEvents } from './IMouseEvents';
+import { LockedMouseEvents } from './LockedMouseEvents';
+import { HoveringMouseEvents } from './HoveringMouseEvents';
+import type { ActiveKeys } from './InputClassesFactory';
+import { EventListenerTracker } from '../Util/EventListenerTracker';
+
+/**
+ * Handles the Mouse Inputs for the document
+ */
+export class MouseController {
+    videoElementProvider: VideoPlayer;
+    toStreamerMessagesProvider: StreamMessageController;
+    coordinateConverter: CoordinateConverter;
+    activeKeysProvider: ActiveKeys;
+
+    // Utility for keeping track of event handlers and unregistering them
+    private mouseEventListenerTracker = new EventListenerTracker();
+
+    /**
+     * @param toStreamerMessagesProvider - Stream message instance
+     * @param videoElementProvider - Video Player instance
+     * @param normalizeAndQuantize - A normalize and quantize instance
+     */
+    constructor(
+        toStreamerMessagesProvider: StreamMessageController,
+        videoElementProvider: VideoPlayer,
+        coordinateConverter: CoordinateConverter,
+        activeKeysProvider: ActiveKeys
+    ) {
+        this.toStreamerMessagesProvider = toStreamerMessagesProvider;
+        this.coordinateConverter = coordinateConverter;
+        this.videoElementProvider = videoElementProvider;
+        this.activeKeysProvider = activeKeysProvider;
+        this.registerMouseEnterAndLeaveEvents();
+    }
+
+    /**
+     * Clears all the click events on the current video element parent div
+     */
+    unregisterMouseEvents() {
+        this.mouseEventListenerTracker.unregisterAll();
+    }
+
+    /**
+     * Register a locked mouse class
+     * @param mouseController - a mouse controller instance
+     * @param playerStyleAttributesProvider - a player style attributes instance
+     */
+    registerLockedMouseEvents(mouseController: MouseController) {
+        const videoElementParent =
+            this.videoElementProvider.getVideoParentElement() as HTMLDivElement;
+        const lockedMouseEvents: IMouseEvents = new LockedMouseEvents(
+            this.videoElementProvider,
+            mouseController,
+            this.activeKeysProvider
+        );
+
+        videoElementParent.requestPointerLock =
+            videoElementParent.requestPointerLock ||
+            videoElementParent.mozRequestPointerLock;
+        document.exitPointerLock =
+            document.exitPointerLock || document.mozExitPointerLock;
+
+        // minor hack to alleviate ios not supporting pointerlock
+        if (videoElementParent.requestPointerLock) {
+            const onclick = () => {
+                videoElementParent.requestPointerLock();
+            };
+            videoElementParent.addEventListener('click', onclick);
+            this.mouseEventListenerTracker.addUnregisterCallback(
+                () => videoElementParent.removeEventListener('click', onclick)
+            );
+        }
+
+        const lockStateChangeListener = () =>
+            lockedMouseEvents.lockStateChange();
+        document.addEventListener(
+            'pointerlockchange',
+            lockStateChangeListener,
+            false
+        );
+        document.addEventListener(
+            'mozpointerlockchange',
+            lockStateChangeListener,
+            false
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => document.removeEventListener(
+                'pointerlockchange',
+                lockStateChangeListener,
+                false
+            )
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => document.removeEventListener(
+                'mozpointerlockchange',
+                lockStateChangeListener,
+                false
+            )
+        );
+
+        const onmousedown = (mouseEvent: MouseEvent) =>
+            lockedMouseEvents.handleMouseDown(mouseEvent);
+        const onmouseup = (mouseEvent: MouseEvent) =>
+            lockedMouseEvents.handleMouseUp(mouseEvent);
+        const onwheel = (wheelEvent: WheelEvent) =>
+            lockedMouseEvents.handleMouseWheel(wheelEvent);
+        const ondblclick = (mouseEvent: MouseEvent) =>
+            lockedMouseEvents.handleMouseDouble(mouseEvent);
+        videoElementParent.addEventListener('mousedown', onmousedown);
+        videoElementParent.addEventListener('mouseup', onmouseup);
+        videoElementParent.addEventListener('wheel', onwheel);
+        videoElementParent.addEventListener('dblclick', ondblclick);
+
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mousedown', onmousedown)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mouseup', onmouseup)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('wheel', onwheel)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('dblclick', ondblclick)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => lockedMouseEvents.unregisterMouseEvents()
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(() => {
+            if (
+                document.exitPointerLock &&
+                (document.pointerLockElement === videoElementParent ||
+                    document.mozPointerLockElement === videoElementParent)
+            ) {
+                document.exitPointerLock();
+            }
+        });
+    }
+
+    /**
+     * Register a hovering mouse class
+     * @param mouseController - A mouse controller object
+     */
+    registerHoveringMouseEvents(mouseController: MouseController) {
+        const videoElementParent =
+            this.videoElementProvider.getVideoParentElement() as HTMLDivElement;
+        const hoveringMouseEvents = new HoveringMouseEvents(mouseController);
+
+        const onmousemove = (mouseEvent: MouseEvent) =>
+            hoveringMouseEvents.updateMouseMovePosition(mouseEvent);
+        const onmousedown = (mouseEvent: MouseEvent) =>
+            hoveringMouseEvents.handleMouseDown(mouseEvent);
+        const onmouseup = (mouseEvent: MouseEvent) =>
+            hoveringMouseEvents.handleMouseUp(mouseEvent);
+        const oncontextmenu = (mouseEvent: MouseEvent) =>
+            hoveringMouseEvents.handleContextMenu(mouseEvent);
+        const onwheel = (wheelEvent: WheelEvent) =>
+            hoveringMouseEvents.handleMouseWheel(wheelEvent);
+        const ondblclick = (mouseEvent: MouseEvent) =>
+            hoveringMouseEvents.handleMouseDouble(mouseEvent);
+        videoElementParent.addEventListener('mousemove', onmousemove);
+        videoElementParent.addEventListener('mousedown', onmousedown);
+        videoElementParent.addEventListener('mouseup', onmouseup);
+        videoElementParent.addEventListener('contextmenu', oncontextmenu);
+        videoElementParent.addEventListener('wheel', onwheel);
+        videoElementParent.addEventListener('dblclick', ondblclick);
+
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mousemove', onmousemove)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mousedown', onmousedown)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mouseup', onmouseup)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('contextmenu', oncontextmenu)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('wheel', onwheel)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('dblclick', ondblclick)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => hoveringMouseEvents.unregisterMouseEvents()
+        );
+    }
+
+    /**
+     * Set the mouse enter and mouse leave events
+     */
+    registerMouseEnterAndLeaveEvents() {
+        const videoElementParent =
+            this.videoElementProvider.getVideoParentElement() as HTMLDivElement;
+
+        // Handle when the Mouse has entered the element
+        const onmouseenter = (event: MouseEvent) => {
+            if (!this.videoElementProvider.isVideoReady()) {
+                return;
+            }
+            Logger.Log(Logger.GetStackTrace(), 'Mouse Entered', 6);
+            this.sendMouseEnter();
+            this.pressMouseButtons(event.buttons, event.x, event.y);
+        };
+
+        // Handles when the mouse has left the element
+        const onmouseleave = (event: MouseEvent) => {
+            if (!this.videoElementProvider.isVideoReady()) {
+                return;
+            }
+            Logger.Log(Logger.GetStackTrace(), 'Mouse Left', 6);
+            this.sendMouseLeave();
+            this.releaseMouseButtons(event.buttons, event.x, event.y);
+        };
+        videoElementParent.addEventListener('mouseenter', onmouseenter);
+        videoElementParent.addEventListener('mouseleave', onmouseleave);
+
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mouseenter', onmouseenter)
+        );
+        this.mouseEventListenerTracker.addUnregisterCallback(
+            () => videoElementParent.removeEventListener('mouseleave', onmouseleave)
+        );
+    }
+
+    /**
+     * Handle when a mouse button is released
+     * @param buttons - Mouse Button
+     * @param X - Mouse pointer X coordinate
+     * @param Y - Mouse pointer Y coordinate
+     */
+    releaseMouseButtons(buttons: number, X: number, Y: number) {
+        const coord = this.coordinateConverter.normalizeAndQuantizeUnsigned(
+            X,
+            Y
+        );
+        if (buttons & MouseButtonsMask.primaryButton) {
+            this.sendMouseUp(MouseButton.mainButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.secondaryButton) {
+            this.sendMouseUp(MouseButton.secondaryButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.auxiliaryButton) {
+            this.sendMouseUp(MouseButton.auxiliaryButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.fourthButton) {
+            this.sendMouseUp(MouseButton.fourthButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.fifthButton) {
+            this.sendMouseUp(MouseButton.fifthButton, coord.x, coord.y);
+        }
+    }
+
+    /**
+     * Handle when a mouse button is pressed
+     * @param buttons - Mouse Button
+     * @param X - Mouse pointer X coordinate
+     * @param Y - Mouse pointer Y coordinate
+     */
+    pressMouseButtons(buttons: number, X: number, Y: number) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        const coord = this.coordinateConverter.normalizeAndQuantizeUnsigned(
+            X,
+            Y
+        );
+        if (buttons & MouseButtonsMask.primaryButton) {
+            this.sendMouseDown(MouseButton.mainButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.secondaryButton) {
+            this.sendMouseDown(MouseButton.secondaryButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.auxiliaryButton) {
+            this.sendMouseDown(MouseButton.auxiliaryButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.fourthButton) {
+            this.sendMouseDown(MouseButton.fourthButton, coord.x, coord.y);
+        }
+        if (buttons & MouseButtonsMask.fifthButton) {
+            this.sendMouseDown(MouseButton.fifthButton, coord.x, coord.y);
+        }
+    }
+
+    /**
+     * Handles mouse enter
+     */
+    sendMouseEnter() {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        const toStreamerHandlers =
+            this.toStreamerMessagesProvider.toStreamerHandlers;
+        toStreamerHandlers.get('MouseEnter')();
+    }
+
+    /**
+     * Handles mouse Leave
+     */
+    sendMouseLeave() {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        const toStreamerHandlers =
+            this.toStreamerMessagesProvider.toStreamerHandlers;
+        toStreamerHandlers.get('MouseLeave')();
+    }
+
+    /**
+     * Handles when a mouse button is pressed down
+     * @param button - Mouse Button Pressed
+     * @param X  - Mouse X Coordinate
+     * @param Y  - Mouse Y Coordinate
+     */
+    sendMouseDown(button: number, X: number, Y: number) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        Logger.Log(
+            Logger.GetStackTrace(),
+            `mouse button ${button} down at (${X}, ${Y})`,
+            6
+        );
+        const toStreamerHandlers =
+            this.toStreamerMessagesProvider.toStreamerHandlers;
+        toStreamerHandlers.get('MouseDown')([button, X, Y]);
+    }
+
+    /**
+     * Handles when a mouse button is pressed up
+     * @param button - Mouse Button Pressed
+     * @param X  - Mouse X Coordinate
+     * @param Y  - Mouse Y Coordinate
+     */
+    sendMouseUp(button: number, X: number, Y: number) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        Logger.Log(
+            Logger.GetStackTrace(),
+            `mouse button ${button} up at (${X}, ${Y})`,
+            6
+        );
+        const coord = this.coordinateConverter.normalizeAndQuantizeUnsigned(
+            X,
+            Y
+        );
+        const toStreamerHandlers =
+            this.toStreamerMessagesProvider.toStreamerHandlers;
+        toStreamerHandlers.get('MouseUp')([button, coord.x, coord.y]);
+    }
+}

package/src/Inputs/SpecialKeyCodes.ts

@@ -0,0 +1,16 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+/**
+ * Registers the Special Key codes
+ *  Must be kept in sync with JavaScriptKeyCodeToFKey C++ array.
+ * The index of the entry in the array is the special key code given below.
+ */
+export class SpecialKeyCodes {
+    static backSpace = 8;
+    static shift = 16;
+    static control = 17;
+    static alt = 18;
+    static rightShift = 253;
+    static rightControl = 254;
+    static rightAlt = 255;
+}

package/src/Inputs/TouchController.ts

@@ -0,0 +1,208 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { CoordinateConverter } from '../Util/CoordinateConverter';
+import { StreamMessageController } from '../UeInstanceMessage/StreamMessageController';
+import { VideoPlayer } from '../VideoPlayer/VideoPlayer';
+import { ITouchController } from './ITouchController';
+import { EventListenerTracker } from '../Util/EventListenerTracker';
+/**
+ * Handles the Touch input Events
+ */
+export class TouchController implements ITouchController {
+    toStreamerMessagesProvider: StreamMessageController;
+    videoElementProvider: VideoPlayer;
+    coordinateConverter: CoordinateConverter;
+    videoElementParent: HTMLVideoElement;
+    fingers = [9, 8, 7, 6, 5, 4, 3, 2, 1, 0];
+    fingerIds = new Map();
+    maxByteValue = 255;
+
+    // Utility for keeping track of event handlers and unregistering them
+    private touchEventListenerTracker = new EventListenerTracker();
+
+    /**
+     * @param toStreamerMessagesProvider - Stream message instance
+     * @param videoElementProvider - Video Player instance
+     * @param coordinateConverter - A coordinate converter instance
+     */
+    constructor(
+        toStreamerMessagesProvider: StreamMessageController,
+        videoElementProvider: VideoPlayer,
+        coordinateConverter: CoordinateConverter
+    ) {
+        this.toStreamerMessagesProvider = toStreamerMessagesProvider;
+        this.videoElementProvider = videoElementProvider;
+        this.coordinateConverter = coordinateConverter;
+        this.videoElementParent = videoElementProvider.getVideoElement();
+        const ontouchstart = (ev: TouchEvent) =>
+            this.onTouchStart(ev);
+        const ontouchend = (ev: TouchEvent) =>
+            this.onTouchEnd(ev);
+        const ontouchmove = (ev: TouchEvent) =>
+            this.onTouchMove(ev);
+        this.videoElementParent.addEventListener('touchstart', ontouchstart);
+        this.videoElementParent.addEventListener('touchend', ontouchend);
+        this.videoElementParent.addEventListener('touchmove', ontouchmove);
+        this.touchEventListenerTracker.addUnregisterCallback(
+            () => this.videoElementParent.removeEventListener('touchstart', ontouchstart)
+        );
+        this.touchEventListenerTracker.addUnregisterCallback(
+            () => this.videoElementParent.removeEventListener('touchend', ontouchend)
+        );
+        this.touchEventListenerTracker.addUnregisterCallback(
+            () => this.videoElementParent.removeEventListener('touchmove', ontouchmove)
+        );
+        Logger.Log(Logger.GetStackTrace(), 'Touch Events Registered', 6);
+
+        // is this strictly necessary?
+        const preventOnTouchMove = (event: TouchEvent) => {
+            event.preventDefault();
+        };
+        document.addEventListener('touchmove', preventOnTouchMove);
+        this.touchEventListenerTracker.addUnregisterCallback(
+            () => document.removeEventListener('touchmove', preventOnTouchMove)
+        );
+    }
+
+    /**
+     * Unregister all touch events
+     */
+    unregisterTouchEvents() {
+        this.touchEventListenerTracker.unregisterAll();
+    }
+
+    /**
+     * Remember a touch command
+     * @param touch - the touch command
+     */
+    rememberTouch(touch: Touch) {
+        const finger = this.fingers.pop();
+        if (finger === undefined) {
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'exhausted touch identifiers',
+                6
+            );
+        }
+        this.fingerIds.set(touch.identifier, finger);
+    }
+
+    /**
+     * Forgets a touch command
+     * @param touch - the touch command
+     */
+    forgetTouch(touch: Touch) {
+        this.fingers.push(this.fingerIds.get(touch.identifier));
+        // Sort array back into descending order. This means if finger '1' were to lift after finger '0', we would ensure that 0 will be the first index to pop
+        this.fingers.sort(function (a, b) {
+            return b - a;
+        });
+        this.fingerIds.delete(touch.identifier);
+    }
+
+    /**
+     * When a touch event starts
+     * @param touchEvent - the touch event being intercepted
+     */
+    onTouchStart(touchEvent: TouchEvent) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        for (let t = 0; t < touchEvent.changedTouches.length; t++) {
+            this.rememberTouch(touchEvent.changedTouches[t]);
+        }
+        Logger.Log(Logger.GetStackTrace(), 'touch start', 6);
+
+        this.emitTouchData('TouchStart', touchEvent.changedTouches);
+        touchEvent.preventDefault();
+    }
+
+    /**
+     * When a touch event ends
+     * @param touchEvent - the touch event being intercepted
+     */
+    onTouchEnd(touchEvent: TouchEvent) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        Logger.Log(Logger.GetStackTrace(), 'touch end', 6);
+        this.emitTouchData('TouchEnd', touchEvent.changedTouches);
+        // Re-cycle unique identifiers previously assigned to each touch.
+        for (let t = 0; t < touchEvent.changedTouches.length; t++) {
+            this.forgetTouch(touchEvent.changedTouches[t]);
+        }
+        touchEvent.preventDefault();
+    }
+
+    /**
+     * when a moving touch event occurs
+     * @param touchEvent - the touch event being intercepted
+     */
+    onTouchMove(touchEvent: TouchEvent) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        Logger.Log(Logger.GetStackTrace(), 'touch move', 6);
+        this.emitTouchData('TouchMove', touchEvent.touches);
+        touchEvent.preventDefault();
+    }
+
+    emitTouchData(type: string, touches: TouchList) {
+        if (!this.videoElementProvider.isVideoReady()) {
+            return;
+        }
+        const offset = this.videoElementProvider.getVideoParentElement().getBoundingClientRect();
+        const toStreamerHandlers =
+            this.toStreamerMessagesProvider.toStreamerHandlers;
+
+        for (let t = 0; t < touches.length; t++) {
+            const numTouches = 1; // the number of touches to be sent this message
+            const touch = touches[t];
+            const x = touch.clientX - offset.left;
+            const y = touch.clientY - offset.top;
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `F${this.fingerIds.get(touch.identifier)}=(${x}, ${y})`,
+                6
+            );
+
+            const coord = this.coordinateConverter.normalizeAndQuantizeUnsigned(
+                x,
+                y
+            );
+            switch (type) {
+                case 'TouchStart':
+                    toStreamerHandlers.get('TouchStart')([
+                        numTouches,
+                        coord.x,
+                        coord.y,
+                        this.fingerIds.get(touch.identifier),
+                        this.maxByteValue * touch.force,
+                        coord.inRange ? 1 : 0
+                    ]);
+                    break;
+                case 'TouchEnd':
+                    toStreamerHandlers.get('TouchEnd')([
+                        numTouches,
+                        coord.x,
+                        coord.y,
+                        this.fingerIds.get(touch.identifier),
+                        this.maxByteValue * touch.force,
+                        coord.inRange ? 1 : 0
+                    ]);
+                    break;
+                case 'TouchMove':
+                    toStreamerHandlers.get('TouchMove')([
+                        numTouches,
+                        coord.x,
+                        coord.y,
+                        this.fingerIds.get(touch.identifier),
+                        this.maxByteValue * touch.force,
+                        coord.inRange ? 1 : 0
+                    ]);
+                    break;
+            }
+        }
+    }
+}