@yang__yj/pixelstreaming-core 1.0.0 → 1.0.2

package/src/WebSockets/WebSocketController.ts

@@ -0,0 +1,267 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '../Logger/Logger';
+import * as MessageReceive from './MessageReceive';
+import * as MessageSend from './MessageSend';
+import { SignallingProtocol } from './SignallingProtocol';
+
+// declare the new method for the websocket interface
+declare global {
+    interface WebSocket {
+        onmessagebinary?(event?: MessageEvent): void;
+    }
+}
+
+/**
+ * The controller for the WebSocket and all associated methods
+ */
+
+export class WebSocketController {
+    WS_OPEN_STATE = 1;
+    webSocket: WebSocket;
+    onOpen: EventTarget;
+    onClose: EventTarget;
+    signallingProtocol: SignallingProtocol;
+
+    constructor() {
+        this.onOpen = new EventTarget();
+        this.onClose = new EventTarget();
+        this.signallingProtocol = new SignallingProtocol();
+        SignallingProtocol.setupDefaultHandlers(this);
+    }
+
+    /**
+     * 连接发信服务器
+     * @param connectionURL——发信服务器的地址
+     * @returns—是否存在连接
+     */
+    connect(connectionURL: string): boolean {
+        Logger.Log(Logger.GetStackTrace(), connectionURL, 6);
+        try {
+            this.webSocket = new WebSocket(connectionURL);
+            this.webSocket.onopen = (event) => this.handleOnOpen(event);
+            this.webSocket.onerror = () => this.handleOnError();
+            this.webSocket.onclose = (event) => this.handleOnClose(event);
+            this.webSocket.onmessage = (event) => this.handleOnMessage(event);
+            this.webSocket.onmessagebinary = (event) =>
+                this.handleOnMessageBinary(event);
+            return true;
+        } catch (error) {
+            Logger.Error(error, error);
+            return false;
+        }
+    }
+
+    /**
+     * 处理以二进制形式接收消息时发生的情况
+     * @param event - Message Received
+     */
+    handleOnMessageBinary(event: MessageEvent) {
+        // if the event is empty return
+        if (!event || !event.data) {
+            return;
+        }
+
+        // handle the binary and then handle the message
+        event.data
+            .text()
+            .then((messageString: unknown) => {
+                // build a new message
+                const constructedMessage = new MessageEvent(
+                    'messageFromBinary',
+                    {
+                        data: messageString
+                    }
+                );
+
+                // 将新的字符串化事件发送回`onmessage`
+                this.handleOnMessage(constructedMessage);
+            })
+            .catch((error: Error) => {
+                Logger.Error(
+                    Logger.GetStackTrace(),
+                    `从websocket解析二进制blob失败，原因: ${error}`
+                );
+            });
+    }
+
+    /**
+     * 处理接收到消息时发生的事情
+     * @param event - Message Received
+     */
+    handleOnMessage(event: MessageEvent) {
+        // 检查websocket消息是否为二进制，如果是，将其字符串化。
+        if (event.data && event.data instanceof Blob) {
+            this.handleOnMessageBinary(event);
+            return;
+        }
+
+        const message: MessageReceive.MessageRecv = JSON.parse(event.data);
+        Logger.Log(
+            Logger.GetStackTrace(),
+            'received => \n' +
+            JSON.stringify(JSON.parse(event.data), undefined, 4),
+            6
+        );
+
+        // 发送到我们的信号协议来处理传入的消息
+        this.signallingProtocol.handleMessage(message.type, event.data);
+    }
+
+    /**
+     * 处理Websocket打开时的事件
+     * @param event - Not Used
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    handleOnOpen(event: Event) {
+        Logger.Log(
+            Logger.GetStackTrace(),
+            '通过WebSocket连接到信令服务器',
+            6
+        );
+        this.onOpen.dispatchEvent(new Event('open'));
+    }
+
+    /**
+     * 在websocket中处理错误
+     * @param event - Error Payload
+     */
+    handleOnError() {
+        Logger.Error(Logger.GetStackTrace(), 'WebSocket error');
+    }
+
+    /**
+     * 当Websocket关闭时处理
+     * @param event - Close Event
+     */
+    handleOnClose(event: CloseEvent) {
+        console.log(event);
+
+        Logger.Log(
+            Logger.GetStackTrace(),
+            '通过WebSocket与信令服务器断开连接: ' +
+            JSON.stringify(event.code) +
+            ' - ' +
+            event.reason
+        );
+
+        // 被踢下线
+        if (event.code == 4004) {
+            this.onOpen.dispatchEvent(new Event('close', event));
+            this.onClose.dispatchEvent(new CustomEvent('close', { 'detail': event }));
+        }
+
+    }
+
+    requestStreamerList() {
+        const payload = new MessageSend.MessageListStreamers();
+        this.webSocket.send(payload.payload());
+    }
+
+    requestStreamerObject(streamerid: string) {
+        const payload = new MessageSend.requestStreamerObject(streamerid);
+        this.webSocket.send(payload.payload());
+    }
+
+    sendSubscribe(streamerid: string) {
+        const payload = new MessageSend.MessageSubscribe(streamerid);
+        this.webSocket.send(payload.payload());
+    }
+
+    sendUnsubscribe() {
+        const payload = new MessageSend.MessageUnsubscribe();
+        this.webSocket.send(payload.payload());
+    }
+
+    sendWebRtcOffer(offer: RTCSessionDescriptionInit) {
+        const payload = new MessageSend.MessageWebRTCOffer(offer);
+        this.webSocket.send(payload.payload());
+    }
+
+    sendWebRtcAnswer(answer: RTCSessionDescriptionInit) {
+        const payload = new MessageSend.MessageWebRTCAnswer(answer);
+        this.webSocket.send(payload.payload());
+    }
+
+    sendWebRtcDatachannelRequest() {
+        const payload = new MessageSend.MessageWebRTCDatachannelRequest();
+        this.webSocket.send(payload.payload());
+    }
+
+    sendSFURecvDataChannelReady() {
+        const payload = new MessageSend.MessageSFURecvDataChannelReady();
+        this.webSocket.send(payload.payload());
+    }
+
+    /**
+    *向服务器发送一个RTC Ice候选项
+    * @param candidate—RTC Ice候选
+     */
+    sendIceCandidate(candidate: RTCIceCandidate) {
+        Logger.Log(Logger.GetStackTrace(), 'Sending Ice Candidate(发送Ice候选人)');
+        if (
+            this.webSocket &&
+            this.webSocket.readyState === this.WS_OPEN_STATE
+        ) {
+            //ws.send(JSON.stringify({ type: 'iceCandidate', candidate: candidate }));
+            const IceCandidate = new MessageSend.MessageIceCandidate(candidate);
+
+            this.webSocket.send(IceCandidate.payload());
+        }
+    }
+
+    /**
+     * Closes the Websocket connection
+     */
+    close() {
+        this.webSocket?.close();
+    }
+
+    /**
+     * The Message Contains the payload of the peer connection options used for the RTC Peer hand shake
+     * @param messageConfig - Config Message received from he signaling server
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onConfig(messageConfig: MessageReceive.MessageConfig) { }
+
+    /**
+     * The Message Contains the payload of the peer connection options used for the RTC Peer hand shake
+     * @param messageConfig - Config Message received from he signaling server
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onStreamerList(messageStreamerList: MessageReceive.MessageStreamerList) { }
+
+    /**
+     * @param iceCandidate - Ice Candidate sent from the Signaling server server's RTC hand shake
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onIceCandidate(iceCandidate: RTCIceCandidateInit) { }
+
+    /**
+     * Event is fired when the websocket receives the answer for the RTC peer Connection
+     * @param messageAnswer - The RTC Answer payload from the signaling server
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onWebRtcAnswer(messageAnswer: MessageReceive.MessageAnswer) { }
+
+    /**
+     * Event is fired when the websocket receives the offer for the RTC peer Connection
+     * @param messageOffer - The sdp offer
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onWebRtcOffer(messageOffer: MessageReceive.MessageOffer) { }
+
+    /**
+     * Event is fired when the websocket receives the data channels for the RTC peer Connection from the SFU
+     * @param messageDataChannels - The data channels details
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onWebRtcPeerDataChannels(messageDataChannels: MessageReceive.MessagePeerDataChannels) { }
+
+    /**
+     * Event is fired when the websocket receives the an updated player count from cirrus
+     * @param MessagePlayerCount - The new player count
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
+    onPlayerCount(playerCount: MessageReceive.MessagePlayerCount) { }
+}

package/src/WebXR/WebXRController.ts

@@ -0,0 +1,319 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '../Logger/Logger';
+import { WebRtcPlayerController } from '../WebRtcPlayer/WebRtcPlayerController';
+import { WebGLUtils } from '../Util/WebGLUtils';
+import { Controller } from '../Inputs/GamepadTypes';
+import { XRGamepadController } from '../Inputs/XRGamepadController';
+import { XrFrameEvent } from '../Util/EventEmitter'
+import { Flags } from '../pixelstreamingfrontend';
+
+export class WebXRController {
+    private xrSession: XRSession;
+    private xrRefSpace: XRReferenceSpace;
+    private gl: WebGL2RenderingContext;
+
+    private positionLocation: number;
+    private texcoordLocation: number;
+    private resolutionLocation: WebGLUniformLocation;
+    private offsetLocation: WebGLUniformLocation;
+
+    private positionBuffer: WebGLBuffer;
+    private texcoordBuffer: WebGLBuffer;
+
+    private webRtcController: WebRtcPlayerController;
+    private xrGamepadController: XRGamepadController;
+    private xrControllers: Array<Controller>;
+
+    onSessionStarted: EventTarget;
+    onSessionEnded: EventTarget;
+    onFrame: EventTarget;
+
+    constructor(webRtcPlayerController: WebRtcPlayerController) {
+        this.xrSession = null;
+        this.webRtcController = webRtcPlayerController;
+        this.xrControllers = [];
+        this.xrGamepadController = new XRGamepadController(
+            this.webRtcController.streamMessageController
+        );
+        this.onSessionEnded = new EventTarget();
+        this.onSessionStarted = new EventTarget();
+        this.onFrame = new EventTarget();
+    }
+
+    public xrClicked() {
+        if (!this.xrSession) {
+            navigator.xr
+                .requestSession('immersive-vr')
+                .then((session: XRSession) => {
+                    this.onXrSessionStarted(session);
+                });
+        } else {
+            this.xrSession.end();
+        }
+    }
+
+    onXrSessionEnded() {
+        Logger.Log(Logger.GetStackTrace(), 'XR Session ended');
+        this.xrSession = null;
+        this.onSessionEnded.dispatchEvent(new Event('xrSessionEnded'));
+    }
+
+    onXrSessionStarted(session: XRSession) {
+        Logger.Log(Logger.GetStackTrace(), 'XR Session started');
+
+        this.xrSession = session;
+        this.xrSession.addEventListener('end', () => {
+            this.onXrSessionEnded();
+        });
+
+        const canvas = document.createElement('canvas');
+        this.gl = canvas.getContext('webgl2', {
+            xrCompatible: true
+        });
+
+        this.xrSession.updateRenderState({
+            baseLayer: new XRWebGLLayer(this.xrSession, this.gl)
+        });
+
+        // setup vertex shader
+        const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+        this.gl.shaderSource(vertexShader, WebGLUtils.vertexShader());
+        this.gl.compileShader(vertexShader);
+
+        // setup fragment shader
+        const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+        this.gl.shaderSource(fragmentShader, WebGLUtils.fragmentShader());
+        this.gl.compileShader(fragmentShader);
+
+        // setup GLSL program
+        const shaderProgram = this.gl.createProgram();
+        this.gl.attachShader(shaderProgram, vertexShader);
+        this.gl.attachShader(shaderProgram, fragmentShader);
+        this.gl.linkProgram(shaderProgram);
+        this.gl.useProgram(shaderProgram);
+
+        // look up where vertex data needs to go
+        this.positionLocation = this.gl.getAttribLocation(
+            shaderProgram,
+            'a_position'
+        );
+        this.texcoordLocation = this.gl.getAttribLocation(
+            shaderProgram,
+            'a_texCoord'
+        );
+        // Create a buffer to put three 2d clip space points in
+        this.positionBuffer = this.gl.createBuffer();
+        // Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.positionBuffer);
+
+        // Turn on the position attribute
+        this.gl.enableVertexAttribArray(this.positionLocation);
+        // Create a texture.
+        const texture = this.gl.createTexture();
+        this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+        // Set the parameters so we can render any size image.
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_WRAP_S,
+            this.gl.CLAMP_TO_EDGE
+        );
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_WRAP_T,
+            this.gl.CLAMP_TO_EDGE
+        );
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_MIN_FILTER,
+            this.gl.NEAREST
+        );
+        this.gl.texParameteri(
+            this.gl.TEXTURE_2D,
+            this.gl.TEXTURE_MAG_FILTER,
+            this.gl.NEAREST
+        );
+
+        this.texcoordBuffer = this.gl.createBuffer();
+        // lookup uniforms
+        this.resolutionLocation = this.gl.getUniformLocation(
+            shaderProgram,
+            'u_resolution'
+        );
+        this.offsetLocation = this.gl.getUniformLocation(
+            shaderProgram,
+            'u_offset'
+        );
+
+        session.requestReferenceSpace('local').then((refSpace) => {
+            this.xrRefSpace = refSpace;
+            this.xrSession.requestAnimationFrame(
+                (time: DOMHighResTimeStamp, frame: XRFrame) =>
+                    this.onXrFrame(time, frame)
+            );
+        });
+
+        this.onSessionStarted.dispatchEvent(new Event('xrSessionStarted'));
+    }
+
+    onXrFrame(time: DOMHighResTimeStamp, frame: XRFrame) {
+        const pose = frame.getViewerPose(this.xrRefSpace);
+        if (pose) {
+            const matrix = pose.transform.matrix;
+            const mat = [];
+            for (let i = 0; i < 16; i++) {
+                mat[i] = new Float32Array([matrix[i]])[0];
+            }
+
+            // prettier-ignore
+            this.webRtcController.streamMessageController.toStreamerHandlers.get('XRHMDTransform')([
+                mat[0], mat[4], mat[8], mat[12],
+                mat[1], mat[5], mat[9], mat[13], 
+                mat[2], mat[6], mat[10], mat[14], 
+                mat[3], mat[7], mat[11], mat[15]
+            ]);
+
+            const glLayer = this.xrSession.renderState.baseLayer;
+            // If we do have a valid pose, bind the WebGL layer's framebuffer,
+            // which is where any content to be displayed on the XRDevice must be
+            // rendered.
+            this.gl.bindFramebuffer(this.gl.FRAMEBUFFER, glLayer.framebuffer);
+
+            // Upload the image into the texture. WebGL knows how to extract the current frame from the video element
+            this.gl.texImage2D(
+                this.gl.TEXTURE_2D,
+                0,
+                this.gl.RGBA,
+                this.gl.RGBA,
+                this.gl.UNSIGNED_BYTE,
+                this.webRtcController.videoPlayer.getVideoElement()
+            );
+            this.render(this.webRtcController.videoPlayer.getVideoElement());
+        }
+
+        if (this.webRtcController.config.isFlagEnabled(Flags.XRControllerInput)) {
+            this.xrSession.inputSources.forEach(
+                (source: XRInputSource, index: number, array: XRInputSource[]) => {
+                    this.xrGamepadController.updateStatus(
+                        source,
+                        frame,
+                        this.xrRefSpace
+                    );
+                },
+                this
+            );
+        }
+
+        this.xrSession.requestAnimationFrame(
+            (time: DOMHighResTimeStamp, frame: XRFrame) =>
+                this.onXrFrame(time, frame)
+        );
+
+        this.onFrame.dispatchEvent(new XrFrameEvent({
+            time,
+            frame
+        }));
+    }
+
+    private render(videoElement: HTMLVideoElement) {
+        if (!this.gl) {
+            return;
+        }
+
+        const glLayer = this.xrSession.renderState.baseLayer;
+        this.gl.viewport(
+            0,
+            0,
+            glLayer.framebufferWidth,
+            glLayer.framebufferHeight
+        );
+        this.gl.uniform4f(this.offsetLocation, 1.0, 1.0, 0.0, 0.0);
+
+        // Set rectangle
+        // prettier-ignore
+        this.gl.bufferData(
+            this.gl.ARRAY_BUFFER,
+            new Float32Array([
+                0, 0, 
+                videoElement.videoWidth, 0,
+                0, videoElement.videoHeight, 
+                0, videoElement.videoHeight,
+                videoElement.videoWidth, 0,
+                videoElement.videoWidth, videoElement.videoHeight
+            ]),
+            this.gl.STATIC_DRAW
+        );
+
+        // Provide texture coordinates for the rectangle
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texcoordBuffer);
+        this.gl.bufferData(
+            this.gl.ARRAY_BUFFER,
+            new Float32Array([
+                0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0
+            ]),
+            this.gl.STATIC_DRAW
+        );
+
+        let size; // components per iteration
+        let type; // the data type
+        let normalize; // normalize the data
+        let stride; // 0 = move forward size * sizeof(type) each iteration to get the next position
+        let offset; // start position of the buffer
+
+        // Bind the position buffer.
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.positionBuffer);
+        // Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
+        size = 2; // 2 components per iteration
+        type = this.gl.FLOAT; // the data is 32bit floats
+        normalize = false; // don't normalize the data
+        stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
+        offset = 0; // start at the beginning of the buffer
+        this.gl.vertexAttribPointer(
+            this.positionLocation,
+            size,
+            type,
+            normalize,
+            stride,
+            offset
+        );
+        // Turn on the texcoord attribute
+        this.gl.enableVertexAttribArray(this.texcoordLocation);
+        // bind the texcoord buffer.
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texcoordBuffer);
+        // Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
+        size = 2; // 2 components per iteration
+        type = this.gl.FLOAT; // the data is 32bit floats
+        normalize = false; // don't normalize the data
+        stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
+        offset = 0; // start at the beginning of the buffer
+        this.gl.vertexAttribPointer(
+            this.texcoordLocation,
+            size,
+            type,
+            normalize,
+            stride,
+            offset
+        );
+        // set the resolution
+        this.gl.uniform2f(
+            this.resolutionLocation,
+            videoElement.videoWidth,
+            videoElement.videoHeight
+        );
+        // draw the rectangle.
+        const primitiveType = this.gl.TRIANGLES;
+        const count = 6;
+        offset = 0;
+        this.gl.drawArrays(primitiveType, offset, count);
+    }
+
+    static isSessionSupported(mode: XRSessionMode): Promise<boolean> {
+        if (navigator.xr) {
+            return navigator.xr.isSessionSupported(mode);
+        } else {
+            return new Promise<boolean>(() => {
+                return false;
+            });
+        }
+    }
+}