@yang__yj/pixelstreaming-core 1.0.0 → 1.0.1

package/src/Util/FileUtil.ts

@@ -0,0 +1,140 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Logger } from '../Logger/Logger';
+
+/**
+ * Utility function for populate file information from byte buffers.
+ */
+export class FileUtil {
+    /**
+     * Processes a files extension when received over data channel
+     * @param view - the file extension data
+     */
+    static setExtensionFromBytes(view: Uint8Array, file: FileTemplate) {
+        // Reset file if we got a file message and we are not "receiving" it yet
+        if (!file.receiving) {
+            file.mimetype = '';
+            file.extension = '';
+            file.receiving = true;
+            file.valid = false;
+            file.size = 0;
+            file.data = [];
+            file.timestampStart = new Date().getTime();
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'Received first chunk of file',
+                6
+            );
+        }
+
+        const extensionAsString = new TextDecoder('utf-16').decode(
+            view.slice(1)
+        );
+        Logger.Log(Logger.GetStackTrace(), extensionAsString, 6);
+        file.extension = extensionAsString;
+    }
+
+    /**
+     * Processes a files mime type when received over data channel
+     * @param view - the file mime type data
+     */
+    static setMimeTypeFromBytes(view: Uint8Array, file: FileTemplate) {
+        // Reset file if we got a file message and we are not "receiving" it yet
+        if (!file.receiving) {
+            file.mimetype = '';
+            file.extension = '';
+            file.receiving = true;
+            file.valid = false;
+            file.size = 0;
+            file.data = [];
+            file.timestampStart = new Date().getTime();
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'Received first chunk of file',
+                6
+            );
+        }
+
+        const mimeAsString = new TextDecoder('utf-16').decode(view.slice(1));
+        Logger.Log(Logger.GetStackTrace(), mimeAsString, 6);
+        file.mimetype = mimeAsString;
+    }
+
+    /**
+     * Processes a files contents when received over data channel
+     * @param view - the file contents data
+     */
+    static setContentsFromBytes(view: Uint8Array, file: FileTemplate) {
+        // If we haven't received the initial setup instructions, return
+        if (!file.receiving) return;
+
+        // Extract the total size of the file (across all chunks)
+        file.size = Math.ceil(
+            new DataView(view.slice(1, 5).buffer).getInt32(0, true) /
+                16379 /* The maximum number of payload bits per message*/
+        );
+
+        // Get the file part of the payload
+        const fileBytes = view.slice(1 + 4);
+
+        // Append to existing data that holds the file
+        file.data.push(fileBytes);
+
+        // Uncomment for debug
+        Logger.Log(
+            Logger.GetStackTrace(),
+            `Received file chunk: ${file.data.length}/${file.size}`,
+            6
+        );
+
+        if (file.data.length === file.size) {
+            file.receiving = false;
+            file.valid = true;
+            Logger.Log(Logger.GetStackTrace(), 'Received complete file', 6);
+            const transferDuration = new Date().getTime() - file.timestampStart;
+            const transferBitrate = Math.round(
+                (file.size * 16 * 1024) / transferDuration
+            );
+            Logger.Log(
+                Logger.GetStackTrace(),
+                `Average transfer bitrate: ${transferBitrate}kb/s over ${
+                    transferDuration / 1000
+                } seconds`,
+                6
+            );
+
+            // File reconstruction
+            /**
+             * Example code to reconstruct the file
+             *
+             * This code reconstructs the received data into the original file based on the mime type and extension provided and then downloads the reconstructed file
+             */
+            const received = new Blob(file.data, { type: file.mimetype });
+            const a = document.createElement('a');
+            a.setAttribute('href', URL.createObjectURL(received));
+            a.setAttribute('download', `transfer.${file.extension}`);
+            document.body.append(a);
+            // if you are so inclined to make it auto-download, do something like: a.click();
+            a.remove();
+        } else if (file.data.length > file.size) {
+            file.receiving = false;
+            Logger.Error(
+                Logger.GetStackTrace(),
+                `Received bigger file than advertised: ${file.data.length}/${file.size}`
+            );
+        }
+    }
+}
+
+/**
+ * A class that represents a template for a downloaded file
+ */
+export class FileTemplate {
+    mimetype = '';
+    extension = '';
+    receiving = false;
+    size = 0;
+    data: Array<Uint8Array> = [];
+    valid = false;
+    timestampStart: number;
+}

package/src/Util/RTCUtils.ts

@@ -0,0 +1,41 @@
+export class RTCUtils {
+    static isVideoTransciever(transceiver : RTCRtpTransceiver | undefined) : boolean {
+        return this.canTransceiverReceiveVideo(transceiver) || this.canTransceiverSendVideo(transceiver);
+    }
+
+    static canTransceiverReceiveVideo(transceiver : RTCRtpTransceiver | undefined) : boolean {
+        return !!transceiver &&
+            (transceiver.direction === 'sendrecv' || transceiver.direction === 'recvonly') &&
+            transceiver.receiver &&
+            transceiver.receiver.track &&
+            transceiver.receiver.track.kind === 'video';    
+    }
+
+    static canTransceiverSendVideo(transceiver : RTCRtpTransceiver | undefined) : boolean {
+        return !!transceiver &&
+            (transceiver.direction === 'sendrecv' || transceiver.direction === 'sendonly') &&
+            transceiver.sender &&
+            transceiver.sender.track &&
+            transceiver.sender.track.kind === 'video';    
+    }
+
+    static isAudioTransciever(transceiver : RTCRtpTransceiver | undefined) : boolean {
+        return this.canTransceiverReceiveAudio(transceiver) || this.canTransceiverSendAudio(transceiver);
+    }
+
+    static canTransceiverReceiveAudio(transceiver : RTCRtpTransceiver | undefined) : boolean {
+        return !!transceiver &&
+            (transceiver.direction === 'sendrecv' || transceiver.direction === 'recvonly') &&
+            transceiver.receiver &&
+            transceiver.receiver.track &&
+            transceiver.receiver.track.kind === 'audio';    
+    }
+
+    static canTransceiverSendAudio(transceiver : RTCRtpTransceiver | undefined) : boolean {
+        return !!transceiver &&
+            (transceiver.direction === 'sendrecv' || transceiver.direction === 'sendonly') &&
+            transceiver.sender &&
+            transceiver.sender.track &&
+            transceiver.sender.track.kind === 'audio';
+    }
+}

package/src/Util/WebGLUtils.ts

@@ -0,0 +1,49 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+export class WebGLUtils {
+    static vertexShader(): string {
+        return `
+		attribute vec2 a_position;
+		attribute vec2 a_texCoord;
+
+		// input
+		uniform vec2 u_resolution;
+		uniform vec4 u_offset;
+
+		//
+		varying vec2 v_texCoord;
+
+		void main() {
+		   // convert the rectangle from pixels to 0.0 to 1.0
+		   vec2 zeroToOne = a_position / u_resolution;
+
+		   // convert from 0->1 to 0->2
+		   vec2 zeroToTwo = zeroToOne * 2.0;
+
+		   // convert from 0->2 to -1->+1 (clipspace)
+		   vec2 clipSpace = zeroToTwo - 1.0;
+
+		   gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
+		   // pass the texCoord to the fragment shader
+		   // The GPU will interpolate this value between points.
+		   v_texCoord = (a_texCoord * u_offset.xy) + u_offset.zw;
+		}
+		`;
+    }
+
+    static fragmentShader(): string {
+        return `
+		precision mediump float;
+
+		// our texture
+		uniform sampler2D u_image;
+
+		// the texCoords passed in from the vertex shader.
+		varying vec2 v_texCoord;
+
+		void main() {
+		   gl_FragColor = texture2D(u_image, v_texCoord);
+		}
+		`;
+    }
+}

package/src/Util/WebXRUtils.ts

@@ -0,0 +1,25 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+export class WebXRUtils {
+    /**
+     * Deep copies a gamepad's values by first converting it to a JSON object and then back to a gamepad
+     *
+     * @param gamepad the original gamepad
+     * @returns a new gamepad object, populated with the original gamepads values
+     */
+    static deepCopyGamepad(gamepad: Gamepad): Gamepad {
+        return JSON.parse(
+            JSON.stringify({
+                buttons: gamepad.buttons.map((b) =>
+                    JSON.parse(
+                        JSON.stringify({
+                            pressed: b.pressed,
+                            touched: b.touched
+                        })
+                    )
+                ),
+                axes: gamepad.axes
+            })
+        );
+    }
+}

package/src/VideoPlayer/StreamController.ts

@@ -0,0 +1,89 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { MouseController } from '../Inputs/MouseController';
+import { Logger } from '../Logger/Logger';
+import { VideoPlayer } from './VideoPlayer';
+
+/**
+ * 视频播放器控制器处理视频HTML元素和所有处理程序的创建
+ */
+export class StreamController {
+    videoElementProvider: VideoPlayer;
+    audioElement: HTMLAudioElement;
+    mouseController: MouseController;
+
+    /**
+     * @param videoElementProvider Video Player instance
+     */
+    constructor(videoElementProvider: VideoPlayer) {
+        this.videoElementProvider = videoElementProvider;
+        this.audioElement = document.createElement('Audio') as HTMLAudioElement;
+        this.videoElementProvider.setAudioElement(this.audioElement);
+    }
+
+    /**
+     * Handles when the Peer connection has a track event
+     * @param rtcTrackEvent - RTC Track Event
+     */
+    handleOnTrack(rtcTrackEvent: RTCTrackEvent) {
+        Logger.Log(
+            Logger.GetStackTrace(),
+            'handleOnTrack ' + JSON.stringify(rtcTrackEvent.streams),
+            6
+        );
+        const videoElement = this.videoElementProvider.getVideoElement();
+
+        if (rtcTrackEvent.track) {
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'Got track - ' +
+                rtcTrackEvent.track.kind +
+                ' id=' +
+                rtcTrackEvent.track.id +
+                ' readyState=' +
+                rtcTrackEvent.track.readyState,
+                6
+            );
+        }
+
+        if (rtcTrackEvent.track.kind == 'audio') {
+            this.CreateAudioTrack(rtcTrackEvent.streams[0]);
+            return;
+        } else if (
+            rtcTrackEvent.track.kind == 'video' &&
+            videoElement.srcObject !== rtcTrackEvent.streams[0]
+        ) {
+            videoElement.srcObject = rtcTrackEvent.streams[0];
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'Set video source from video track ontrack.'
+            );
+            return;
+        }
+    }
+
+    /**
+     * 当接收到类型为"audio"的RTCTrackEvent时创建音频设备
+     * @param audioMediaStream—音频媒体流track
+     */
+    CreateAudioTrack(audioMediaStream: MediaStream) {
+        const videoElement = this.videoElementProvider.getVideoElement();
+
+        // 什么都不做视频和音频具有相同的媒体流(它们是链接的)
+        if (videoElement.srcObject == audioMediaStream) {
+            return;
+        }
+        // Video元素有一些与此音轨无关的其他媒体流
+        else if (
+            videoElement.srcObject &&
+            videoElement.srcObject !== audioMediaStream
+        ) {
+            // create a new audio element
+            this.audioElement.srcObject = audioMediaStream;
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'Created new audio element to play separate audio stream.'
+            );
+        }
+    }
+}

package/src/VideoPlayer/VideoPlayer.ts

@@ -0,0 +1,246 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import { Config, Flags } from '../Config/Config';
+import { Logger } from '../Logger/Logger';
+
+/**
+ * Extra types for the HTMLElement
+ */
+declare global {
+    interface HTMLElement {
+        mozRequestPointerLock?(): void;
+    }
+}
+
+/**
+ * The video player html element
+ */
+export class VideoPlayer {
+    private config: Config;
+    private videoElement: HTMLVideoElement;
+    private audioElement?: HTMLAudioElement;
+    private orientationChangeTimeout: number;
+    private lastTimeResized = new Date().getTime();
+
+    onMatchViewportResolutionCallback: (width: number, height: number) => void;
+    onResizePlayerCallback: () => void;
+    resizeTimeoutHandle: number;
+
+    /**
+     * @param videoElementParent the html div the the video player will be injected into
+     * @param config the applications configuration. We're interested in the startVideoMuted flag
+     */
+    constructor(videoElementParent: HTMLElement, config: Config) {
+        this.videoElement = document.createElement('video');
+        this.config = config;
+        this.videoElement.id = 'streamingVideo';
+        this.videoElement.disablePictureInPicture = true;
+        this.videoElement.playsInline = true;
+        this.videoElement.style.width = '100%';
+        this.videoElement.style.height = '100%';
+        this.videoElement.style.position = 'absolute';
+        this.videoElement.style.pointerEvents = 'all';
+        videoElementParent.appendChild(this.videoElement);
+
+        this.onResizePlayerCallback = () => {
+            console.log(
+                'Resolution changed, restyling player, did you forget to override this function?'
+            );
+        };
+        this.onMatchViewportResolutionCallback = () => {
+            console.log(
+                'Resolution changed and match viewport resolution is turned on, did you forget to override this function?'
+            );
+        };
+
+        // set play for video (and audio)
+        this.videoElement.onclick = () => {
+            if (this.audioElement != undefined && this.audioElement.paused) {
+                this.audioElement.play();
+            }
+            if (this.videoElement.paused) {
+                this.videoElement.play();
+            }
+        };
+
+        this.videoElement.onloadedmetadata = () => {
+            this.onVideoInitialized();
+        };
+
+        // set resize events to the windows if it is resized or its orientation is changed
+        window.addEventListener('resize', () => this.resizePlayerStyle(), true);
+        window.addEventListener('orientationchange', () =>
+            this.onOrientationChange()
+        );
+    }
+
+    public setAudioElement(audioElement: HTMLAudioElement) : void {
+        this.audioElement = audioElement;
+    }
+
+    /**
+     * Sets up the video element with any application config and plays the video element.
+     * @returns A promise for if playing the video was successful or not.
+     */
+    play(): Promise<void> {
+        this.videoElement.muted = this.config.isFlagEnabled(
+            Flags.StartVideoMuted
+        );
+        this.videoElement.autoplay = this.config.isFlagEnabled(
+            Flags.AutoPlayVideo
+        );
+        return this.videoElement.play();
+    }
+
+    /**
+     * @returns True if the video element is paused.
+     */
+    isPaused(): boolean {
+        return this.videoElement.paused;
+    }
+
+    /**
+     * @returns - whether the video element is playing.
+     */
+    isVideoReady(): boolean {
+        return (
+            this.videoElement.readyState !== undefined &&
+            this.videoElement.readyState > 0
+        );
+    }
+
+    /**
+     * @returns True if the video element has a valid video source (srcObject).
+     */
+    hasVideoSource(): boolean {
+        return (
+            this.videoElement.srcObject !== undefined &&
+            this.videoElement.srcObject !== null
+        );
+    }
+
+    /**
+     * Get the current context of the html video element
+     * @returns - the current context of the video element
+     */
+    getVideoElement(): HTMLVideoElement {
+        return this.videoElement;
+    }
+
+    /**
+     * Get the current context of the html video elements parent
+     * @returns - the current context of the video elements parent
+     */
+    getVideoParentElement(): HTMLElement {
+        return this.videoElement.parentElement;
+    }
+
+    /**
+     * Set the Video Elements src object tracks to enable
+     * @param enabled - Enable Tracks on the Src Object
+     */
+    setVideoEnabled(enabled: boolean) {
+        // this is a temporary hack until type scripts video element is updated to reflect the need for tracks on a html video element
+        const videoElement = this.videoElement;
+        (<MediaStream>videoElement.srcObject)
+            .getTracks()
+            .forEach((track: MediaStreamTrack) => (track.enabled = enabled));
+    }
+
+    /**
+     * An override for when the video has been initialized with a srcObject
+     */
+    onVideoInitialized() {
+        // Default Functionality: Do Nothing
+    }
+
+    /**
+     * On the orientation change of a window clear the timeout
+     */
+    onOrientationChange() {
+        clearTimeout(this.orientationChangeTimeout);
+        this.orientationChangeTimeout = window.setTimeout(() => {
+            this.resizePlayerStyle();
+        }, 500);
+    }
+
+    /**
+     * Resizes the player style based on the window height and width
+     * @returns - nil if requirements are satisfied
+     */
+    resizePlayerStyle() {
+        const videoElementParent = this.getVideoParentElement();
+
+        if (!videoElementParent) {
+            return;
+        }
+
+        this.updateVideoStreamSize();
+
+        if (videoElementParent.classList.contains('fixed-size')) {
+            this.onResizePlayerCallback();
+            return;
+        }
+
+        // controls for resizing the player
+        this.resizePlayerStyleToFillParentElement();
+        this.onResizePlayerCallback();
+    }
+
+    /**
+     * Resizes the player element to fill the parent element
+     */
+    resizePlayerStyleToFillParentElement() {
+        const videoElementParent = this.getVideoParentElement();
+
+        //Video is not initialized yet so set videoElementParent to size of parent element
+        const styleWidth = '100%';
+        const styleHeight = '100%';
+        const styleTop = 0;
+        const styleLeft = 0;
+        videoElementParent.setAttribute(
+            'style',
+            'top: ' +
+                styleTop +
+                'px; left: ' +
+                styleLeft +
+                'px; width: ' +
+                styleWidth +
+                '; height: ' +
+                styleHeight +
+                '; cursor: default;'
+        );
+    }
+
+    updateVideoStreamSize() {
+        if (!this.config.isFlagEnabled(Flags.MatchViewportResolution)) {
+            return;
+        }
+
+        const now = new Date().getTime();
+        if (now - this.lastTimeResized > 300) {
+            const videoElementParent = this.getVideoParentElement();
+            if (!videoElementParent) {
+                return;
+            }
+
+            this.onMatchViewportResolutionCallback(
+                videoElementParent.clientWidth,
+                videoElementParent.clientHeight
+            );
+
+            this.lastTimeResized = new Date().getTime();
+        } else {
+            Logger.Log(
+                Logger.GetStackTrace(),
+                'Resizing too often - skipping',
+                6
+            );
+            clearTimeout(this.resizeTimeoutHandle);
+            this.resizeTimeoutHandle = window.setTimeout(
+                () => this.updateVideoStreamSize(),
+                100
+            );
+        }
+    }
+}