@epicgames-ps/lib-pixelstreamingfrontend-ue5.5 0.0.5

package/src/WebXR/WebXRController.ts

@@ -0,0 +1,319 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '@epicgames-ps/lib-pixelstreamingcommon-ue5.5';
+import { WebRtcPlayerController } from '../WebRtcPlayer/WebRtcPlayerController';
+import { WebGLUtils } from '../Util/WebGLUtils';
+import { Controller } from '../Inputs/GamepadTypes';
+import { XRGamepadController } from '../Inputs/XRGamepadController';
+import { XrFrameEvent } from '../Util/EventEmitter'
+import { Flags } from '../pixelstreamingfrontend';
+
+export class WebXRController {
+    private xrSession: XRSession;
+    private xrRefSpace: XRReferenceSpace;
+    private gl: WebGL2RenderingContext;
+
+    private positionLocation: number;
+    private texcoordLocation: number;
+    private resolutionLocation: WebGLUniformLocation;
+    private offsetLocation: WebGLUniformLocation;
+
+    private positionBuffer: WebGLBuffer;
+    private texcoordBuffer: WebGLBuffer;
+
+    private webRtcController: WebRtcPlayerController;
+    private xrGamepadController: XRGamepadController;
+    private xrControllers: Array<Controller>;
+
+    onSessionStarted: EventTarget;
+    onSessionEnded: EventTarget;
+    onFrame: EventTarget;
+
+    constructor(webRtcPlayerController: WebRtcPlayerController) {
+        this.xrSession = null;
+        this.webRtcController = webRtcPlayerController;
+        this.xrControllers = [];
+        this.xrGamepadController = new XRGamepadController(
+            this.webRtcController.streamMessageController
+        );
+        this.onSessionEnded = new EventTarget();
+        this.onSessionStarted = new EventTarget();
+        this.onFrame = new EventTarget();
+    }
+
+    public xrClicked() {
+        if (!this.xrSession) {
+            navigator.xr
+                .requestSession('immersive-vr')
+                .then((session: XRSession) => {
+                    this.onXrSessionStarted(session);
+                });
+        } else {
+            this.xrSession.end();
+        }
+    }
+
+    onXrSessionEnded() {
+        Logger.Log(Logger.GetStackTrace(), 'XR Session ended');
+        this.xrSession = null;
+        this.onSessionEnded.dispatchEvent(new Event('xrSessionEnded'));
+    }
+
+    onXrSessionStarted(session: XRSession) {
+        Logger.Log(Logger.GetStackTrace(), 'XR Session started');
+
+        this.xrSession = session;
+        this.xrSession.addEventListener('end', () => {
+            this.onXrSessionEnded();
+        });
+
+        const canvas = document.createElement('canvas');
+        this.gl = canvas.getContext('webgl2', {
+            xrCompatible: true
+        });
+
+        this.xrSession.updateRenderState({
+            baseLayer: new XRWebGLLayer(this.xrSession, this.gl)
+        });
+
+        // setup vertex shader
+        const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+        this.gl.shaderSource(vertexShader, WebGLUtils.vertexShader());
+        this.gl.compileShader(vertexShader);
+
+        // setup fragment shader
+        const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+        this.gl.shaderSource(fragmentShader, WebGLUtils.fragmentShader());
+        this.gl.compileShader(fragmentShader);
+
+        // setup GLSL program
+        const shaderProgram = this.gl.createProgram();
+        this.gl.attachShader(shaderProgram, vertexShader);
+        this.gl.attachShader(shaderProgram, fragmentShader);
+        this.gl.linkProgram(shaderProgram);
+        this.gl.useProgram(shaderProgram);
+
+        // look up where vertex data needs to go
+        this.positionLocation = this.gl.getAttribLocation(
+            shaderProgram,
+            'a_position'
+        );
+        this.texcoordLocation = this.gl.getAttribLocation(
+            shaderProgram,
+            'a_texCoord'
+        );
+        // Create a buffer to put three 2d clip space points in
+        this.positionBuffer = this.gl.createBuffer();
+        // Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.positionBuffer);
+
+        // Turn on the position attribute
+        this.gl.enableVertexAttribArray(this.positionLocation);
+        // Create a texture.
+        const texture = this.gl.createTexture();
+        this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+        // Set the parameters so we can render any size image.
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_WRAP_S,
+            this.gl.CLAMP_TO_EDGE
+        );
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_WRAP_T,
+            this.gl.CLAMP_TO_EDGE
+        );
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_MIN_FILTER,
+            this.gl.NEAREST
+        );
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_MAG_FILTER,
+            this.gl.NEAREST
+        );
+
+        this.texcoordBuffer = this.gl.createBuffer();
+        // lookup uniforms
+        this.resolutionLocation = this.gl.getUniformLocation(
+            shaderProgram,
+            'u_resolution'
+        );
+        this.offsetLocation = this.gl.getUniformLocation(
+            shaderProgram,
+            'u_offset'
+        );
+
+        session.requestReferenceSpace('local').then((refSpace) => {
+            this.xrRefSpace = refSpace;
+            this.xrSession.requestAnimationFrame(
+                (time: DOMHighResTimeStamp, frame: XRFrame) =>
+                    this.onXrFrame(time, frame)
+            );
+        });
+
+        this.onSessionStarted.dispatchEvent(new Event('xrSessionStarted'));
+    }
+
+    onXrFrame(time: DOMHighResTimeStamp, frame: XRFrame) {
+        const pose = frame.getViewerPose(this.xrRefSpace);
+        if (pose) {
+            const matrix = pose.transform.matrix;
+            const mat = [];
+            for (let i = 0; i < 16; i++) {
+                mat[i] = new Float32Array([matrix[i]])[0];
+            }
+
+            // prettier-ignore
+            this.webRtcController.streamMessageController.toStreamerHandlers.get('XRHMDTransform')([
+                mat[0], mat[4], mat[8], mat[12],
+                mat[1], mat[5], mat[9], mat[13], 
+                mat[2], mat[6], mat[10], mat[14], 
+                mat[3], mat[7], mat[11], mat[15]
+            ]);
+
+            const glLayer = this.xrSession.renderState.baseLayer;
+            // If we do have a valid pose, bind the WebGL layer's framebuffer,
+            // which is where any content to be displayed on the XRDevice must be
+            // rendered.
+            this.gl.bindFramebuffer(this.gl.FRAMEBUFFER, glLayer.framebuffer);
+
+            // Upload the image into the texture. WebGL knows how to extract the current frame from the video element
+            this.gl.texImage2D(
+                this.gl.TEXTURE_2D,
+                0,
+                this.gl.RGBA,
+                this.gl.RGBA,
+                this.gl.UNSIGNED_BYTE,
+                this.webRtcController.videoPlayer.getVideoElement()
+            );
+            this.render(this.webRtcController.videoPlayer.getVideoElement());
+        }
+
+        if (this.webRtcController.config.isFlagEnabled(Flags.XRControllerInput)) {
+            this.xrSession.inputSources.forEach(
+                (source: XRInputSource, index: number, array: XRInputSource[]) => {
+                    this.xrGamepadController.updateStatus(
+                        source,
+                        frame,
+                        this.xrRefSpace
+                    );
+                },
+                this
+            );
+        }
+
+        this.xrSession.requestAnimationFrame(
+            (time: DOMHighResTimeStamp, frame: XRFrame) =>
+                this.onXrFrame(time, frame)
+        );
+
+        this.onFrame.dispatchEvent(new XrFrameEvent({
+            time,
+            frame
+        }));
+    }
+
+    private render(videoElement: HTMLVideoElement) {
+        if (!this.gl) {
+            return;
+        }
+
+        const glLayer = this.xrSession.renderState.baseLayer;
+        this.gl.viewport(
+            0,
+            0,
+            glLayer.framebufferWidth,
+            glLayer.framebufferHeight
+        );
+        this.gl.uniform4f(this.offsetLocation, 1.0, 1.0, 0.0, 0.0);
+
+        // Set rectangle
+        // prettier-ignore
+        this.gl.bufferData(
+            this.gl.ARRAY_BUFFER,
+            new Float32Array([
+                0, 0, 
+                videoElement.videoWidth, 0,
+                0, videoElement.videoHeight, 
+                0, videoElement.videoHeight,
+                videoElement.videoWidth, 0,
+                videoElement.videoWidth, videoElement.videoHeight
+            ]),
+            this.gl.STATIC_DRAW
+        );
+
+        // Provide texture coordinates for the rectangle
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texcoordBuffer);
+        this.gl.bufferData(
+            this.gl.ARRAY_BUFFER,
+            new Float32Array([
+                0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0
+            ]),
+            this.gl.STATIC_DRAW
+        );
+
+        let size; // components per iteration
+        let type; // the data type
+        let normalize; // normalize the data
+        let stride; // 0 = move forward size * sizeof(type) each iteration to get the next position
+        let offset; // start position of the buffer
+
+        // Bind the position buffer.
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.positionBuffer);
+        // Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
+        size = 2; // 2 components per iteration
+        type = this.gl.FLOAT; // the data is 32bit floats
+        normalize = false; // don't normalize the data
+        stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
+        offset = 0; // start at the beginning of the buffer
+        this.gl.vertexAttribPointer(
+            this.positionLocation,
+            size,
+            type,
+            normalize,
+            stride,
+            offset
+        );
+        // Turn on the texcoord attribute
+        this.gl.enableVertexAttribArray(this.texcoordLocation);
+        // bind the texcoord buffer.
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texcoordBuffer);
+        // Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
+        size = 2; // 2 components per iteration
+        type = this.gl.FLOAT; // the data is 32bit floats
+        normalize = false; // don't normalize the data
+        stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
+        offset = 0; // start at the beginning of the buffer
+        this.gl.vertexAttribPointer(
+            this.texcoordLocation,
+            size,
+            type,
+            normalize,
+            stride,
+            offset
+        );
+        // set the resolution
+        this.gl.uniform2f(
+            this.resolutionLocation,
+            videoElement.videoWidth,
+            videoElement.videoHeight
+        );
+        // draw the rectangle.
+        const primitiveType = this.gl.TRIANGLES;
+        const count = 6;
+        offset = 0;
+        this.gl.drawArrays(primitiveType, offset, count);
+    }
+
+    static isSessionSupported(mode: XRSessionMode): Promise<boolean> {
+        if (navigator.xr) {
+            return navigator.xr.isSessionSupported(mode);
+        } else {
+            return new Promise<boolean>(() => {
+                return false;
+            });
+        }
+    }
+}

package/src/__test__/mockMediaStream.ts

@@ -0,0 +1,124 @@
+export class MockMediaStreamImpl implements MediaStream {
+    active: boolean;
+    id: string;
+
+    constructor(data?: MediaStream | MediaStreamTrack[]) {
+        //
+    }
+
+    onaddtrack: ((this: MediaStream, ev: MediaStreamTrackEvent) => any) | null;
+    onremovetrack: ((this: MediaStream, ev: MediaStreamTrackEvent) => any) | null;
+    addTrack(track: MediaStreamTrack): void {
+        throw new Error("Method not implemented.");
+    }
+    clone(): MediaStream {
+        throw new Error("Method not implemented.");
+    }
+    getAudioTracks(): MediaStreamTrack[] {
+        throw new Error("Method not implemented.");
+    }
+    getTrackById(trackId: string): MediaStreamTrack | null {
+        throw new Error("Method not implemented.");
+    }
+    getTracks(): MediaStreamTrack[] {
+        throw new Error("Method not implemented.");
+    }
+    getVideoTracks(): MediaStreamTrack[] {
+        throw new Error("Method not implemented.");
+    }
+    removeTrack(track: MediaStreamTrack): void {
+        throw new Error("Method not implemented.");
+    }
+    addEventListener<K extends keyof MediaStreamEventMap>(type: K, listener: (this: MediaStream, ev: MediaStreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions | undefined): void;
+    addEventListener(type: string, listener: EventListenerOrEventListenerObject, options?: boolean | AddEventListenerOptions | undefined): void;
+    addEventListener(type: unknown, listener: unknown, options?: unknown): void {
+        throw new Error("Method not implemented.");
+    }
+    removeEventListener<K extends keyof MediaStreamEventMap>(type: K, listener: (this: MediaStream, ev: MediaStreamEventMap[K]) => any, options?: boolean | EventListenerOptions | undefined): void;
+    removeEventListener(type: string, listener: EventListenerOrEventListenerObject, options?: boolean | EventListenerOptions | undefined): void;
+    removeEventListener(type: unknown, listener: unknown, options?: unknown): void {
+        throw new Error("Method not implemented.");
+    }
+    dispatchEvent(event: Event): boolean {
+        throw new Error("Method not implemented.");
+    }
+
+}
+
+export class MockMediaStreamTrackImpl implements MediaStreamTrack {
+    contentHint: string;
+    enabled: boolean;
+    id: string;
+    kind: string;
+    label: string;
+    muted: boolean;
+    readyState: MediaStreamTrackState;
+
+    constructor() {
+        this.kind = 'video';
+        this.readyState = 'live';
+    }
+
+    onended: ((this: MediaStreamTrack, ev: Event) => any) | null;
+    onmute: ((this: MediaStreamTrack, ev: Event) => any) | null;
+    onunmute: ((this: MediaStreamTrack, ev: Event) => any) | null;
+    applyConstraints(constraints?: MediaTrackConstraints | undefined): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+    clone(): MediaStreamTrack {
+        throw new Error("Method not implemented.");
+    }
+    getCapabilities(): MediaTrackCapabilities {
+        throw new Error("Method not implemented.");
+    }
+    getConstraints(): MediaTrackConstraints {
+        throw new Error("Method not implemented.");
+    }
+    getSettings(): MediaTrackSettings {
+        throw new Error("Method not implemented.");
+    }
+    stop(): void {
+        throw new Error("Method not implemented.");
+    }
+    addEventListener<K extends keyof MediaStreamTrackEventMap>(type: K, listener: (this: MediaStreamTrack, ev: MediaStreamTrackEventMap[K]) => any, options?: boolean | AddEventListenerOptions | undefined): void;
+    addEventListener(type: string, listener: EventListenerOrEventListenerObject, options?: boolean | AddEventListenerOptions | undefined): void;
+    addEventListener(type: unknown, listener: unknown, options?: unknown): void {
+        throw new Error("Method not implemented.");
+    }
+    removeEventListener<K extends keyof MediaStreamTrackEventMap>(type: K, listener: (this: MediaStreamTrack, ev: MediaStreamTrackEventMap[K]) => any, options?: boolean | EventListenerOptions | undefined): void;
+    removeEventListener(type: string, listener: EventListenerOrEventListenerObject, options?: boolean | EventListenerOptions | undefined): void;
+    removeEventListener(type: unknown, listener: unknown, options?: unknown): void {
+        throw new Error("Method not implemented.");
+    }
+    dispatchEvent(event: Event): boolean {
+        throw new Error("Method not implemented.");
+    }
+}
+
+const mockHTMLMediaElementPlay = (ableToPlay: boolean) => {
+    if (ableToPlay) {
+        return Promise.resolve();
+    }
+    return Promise.reject("mock cancel");
+};
+
+const originalMediaStream = global.MediaStream;
+const originalMediaStreamTrack = global.MediaStreamTrack;
+export const mockMediaStream = () => {
+    global.MediaStream = MockMediaStreamImpl;
+    global.MediaStreamTrack = MockMediaStreamTrackImpl;
+}
+
+export const unmockMediaStream = () => {
+    global.MediaStream = originalMediaStream;
+    global.MediaStreamTrack = originalMediaStreamTrack;
+}
+
+export const mockHTMLMediaElement = (options: { ableToPlay: boolean, readyState?: number }) => {
+    const { ableToPlay, readyState } = options;
+    jest.spyOn(HTMLMediaElement.prototype, 'play').mockReturnValue(mockHTMLMediaElementPlay(ableToPlay));
+    if (readyState !== undefined) {
+        jest.spyOn(HTMLMediaElement.prototype, 'readyState', 'get').mockReturnValue(readyState);
+    }
+}
+