@needle-tools/engine 4.4.0-beta.3 → 4.4.0-beta.5

package/src/engine-components/ScreenCapture.ts

@@ -1,592 +1,592 @@
-import { isDevEnvironment, showBalloonWarning } from "../engine/debug/index.js";
-import { RoomEvents } from "../engine/engine_networking.js";
-import { disposeStream, NetworkedStreamEvents, NetworkedStreams, PeerHandle, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js";
-import { serializable } from "../engine/engine_serialization.js";
-import { delay, getParam } from "../engine/engine_utils.js";
-import { AudioSource } from "./AudioSource.js";
-import { Behaviour, GameObject } from "./Component.js";
-import type { IPointerClickHandler, PointerEventData } from "./ui/PointerEvents.js";
-import { AspectMode, VideoPlayer } from "./VideoPlayer.js";
-
-const debug = getParam("debugscreensharing");
-
-/**
- * ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room.
- */
-export enum ScreenCaptureDevice {
-    /**
-     * Capture the screen of the user.
-     */
-    Screen = 0,
-    /**
-     * Capture the camera of the user.
-     */
-    Camera = 1,
-    /** Please note that canvas streaming might not work reliably on chrome: https://bugs.chromium.org/p/chromium/issues/detail?id=1156408 */
-    Canvas = 2,
-    /** When using Microphone only the voice will be send */
-    Microphone = 3
-}
-
-/**
- * {@link ScreenCapture} allows you to share your screen, camera or microphone with other users in the networked room.
- */
-declare type ScreenCaptureDeviceTypes = keyof typeof ScreenCaptureDevice;
-
-/**
- * The current mode of the {@link ScreenCapture} component.
- */
-export enum ScreenCaptureMode {
-    Idle = 0,
-    Sending = 1,
-    Receiving = 2
-}
-
-/**
- * Options for the {@link ScreenCapture} component when starting to share a stream by calling the {@link ScreenCapture.share}.
- */
-export declare type ScreenCaptureOptions = {
-    /**
-     * You can specify the device type to capture (e.g. Screen, Camera, Microphone)
-     */
-    device?: ScreenCaptureDeviceTypes,
-    /**
-     * Constraints for the media stream like resolution, frame rate, etc.
-     * @see https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamConstraints
-     */
-    constraints?: MediaTrackConstraints,
-    /** Filter video device by id. Alternatively pass in a deviceFilter callback to manually filter available devices */
-    deviceId?: string,
-    /** Return false to skip the available device */
-    deviceFilter?: (device: MediaDeviceInfo) => boolean,
-}
-
-/** 
- * The ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room.   
- * When the stream is active the video will be displayed on the VideoPlayer component attached to the same GameObject.  
- * 
- * Note: For debugging append `?debugscreensharing` to the URL to see more information in the console.
- * 
- * By default the component will start sharing the screen when the user clicks on the object this component is attached to. You can set {@link device} This behaviour can be disabled by setting `allowStartOnClick` to false.  
- * It is also possible to start the stream manually from your code by calling the {@link share} method.
- * 
- * @category Networking
- * @group Components
-*/
-export class ScreenCapture extends Behaviour implements IPointerClickHandler {
-
-    /**
-     * When enabled the stream will start when the user clicks on the object this component is attached to  
-     * It is also possible to start the stream manually from your code by calling the {@link share} method    
-     * To modify what type of device is shared you can set the {@link device} property.
-     * @default true
-     */
-    @serializable()
-    allowStartOnClick: boolean = true;
-
-    /** @internal */
-    onPointerEnter() {
-        if (this.context.connection.allowEditing == false) return;
-        if (!this.allowStartOnClick) return;
-        this.context.input.setCursor("pointer");
-    }
-    /** @internal */
-    onPointerExit() {
-        if (this.context.connection.allowEditing == false) return;
-        if (!this.allowStartOnClick) return;
-        this.context.input.unsetCursor("pointer");
-    }
-
-    /** @internal */
-    onPointerClick(evt: PointerEventData) {
-        if (this.context.connection.allowEditing == false) return;
-        if (!this.allowStartOnClick) return;
-        if (evt && evt.pointerId !== 0) return;
-        if (this.isReceiving && this.videoPlayer?.isPlaying) {
-            if (this.videoPlayer)
-                this.videoPlayer.screenspace = !this.videoPlayer.screenspace;
-            return;
-
-        }
-        if (this.isSending) {
-            this.close();
-            return;
-        }
-        this.share();
-    }
-
-
-    /** When enabled the stream will start when this component becomes active (enabled in the scene) */
-    @serializable()
-    autoConnect: boolean = false;
-
-    /**
-     * If a VideoPlayer component is assigned to this property the video will be displayed on the VideoPlayer component.
-     */
-    @serializable(VideoPlayer)
-    set videoPlayer(val: VideoPlayer | undefined) {
-        if (this._videoPlayer && (this.isSending || this.isReceiving)) {
-            this._videoPlayer.stop();
-        }
-        this._videoPlayer = val;
-        if (this._videoPlayer && this._currentStream && (this.isSending || this.isReceiving)) {
-            this._videoPlayer.setVideo(this._currentStream);
-        }
-    }
-    get videoPlayer() { return this._videoPlayer; }
-    private _videoPlayer?: VideoPlayer;
-    private _audioSource?: AudioSource;
-
-    /**
-     * When enabled the video will be displayed in the screenspace of the VideoPlayer component.
-     */
-    get screenspace() { return this.videoPlayer?.screenspace ?? false; }
-    set screenspace(v: boolean) { if (this.videoPlayer) this.videoPlayer.screenspace = v; }
-
-    /**
-     * Which streaming device type should be used when starting to share (if {@link share} is called without a device option). Options are Screen, Camera, Microphone.  
-     * This is e.g. used if `allowStartOnClick` is enabled and the user clicks on the object. 
-     * @default Screen 
-     */
-    @serializable()
-    device: ScreenCaptureDeviceTypes = "Screen";
-
-    /**
-     * If assigned the device the device will be selected by this id or label when starting to share.  
-     * Note: This is only supported for `Camera` devices
-     */
-    @serializable()
-    deviceName?: string;
-
-    /**
-     * Filter which device should be chosen for sharing by id or label.  
-     * Assign a method to this property to manually filter the available devices.  
-     */
-    deviceFilter?: (device: MediaDeviceInfo) => boolean;
-
-    /**
-     * the current stream that is being shared or received  
-     * @link https://developer.mozilla.org/en-US/docs/Web/API/MediaStream
-     */
-    get currentScream(): MediaStream | null {
-        return this._currentStream;
-    }
-    get currentMode(): ScreenCaptureMode {
-        return this._currentMode;
-    }
-
-    /**
-     * @returns true if the component is currently sending a stream
-     */
-    get isSending() {
-        return this._currentStream?.active && this._currentMode === ScreenCaptureMode.Sending;
-    }
-    /**
-     * @returns true if the component is currently receiving a stream
-     */
-    get isReceiving() {
-        if (this._currentMode === ScreenCaptureMode.Receiving) {
-            if (!this._currentStream || this._currentStream.active === false) return false;
-            // if any track is still live consider it active
-            const tracks = this._currentStream.getTracks();
-            for (const track of tracks) {
-                if (track.readyState === "live") return true;
-            }
-        }
-        return false;
-    }
-
-    private get requiresVideoPlayer() { 
-        return this.device !== "Microphone";
-    }
-    private _net?: NetworkedStreams;
-    private _requestOpen: boolean = false;
-    private _currentStream: MediaStream | null = null;
-    private _currentMode: ScreenCaptureMode = ScreenCaptureMode.Idle;
-
-    /** @internal */
-    awake() {
-        // Resolve the device type if it is a number
-        if (typeof this.device === "number") {
-            this.device = ScreenCaptureDevice[this.device] as ScreenCaptureDeviceTypes;
-        }
-
-        if (debug)
-            console.log("Screensharing", this.name, this);
-        AudioSource.registerWaitForAllowAudio(() => {
-            if (this._videoPlayer && this._currentStream && this._currentMode === ScreenCaptureMode.Receiving) {
-                this._videoPlayer.playInBackground = true;
-                this._videoPlayer.setVideo(this._currentStream);
-            }
-        });
-        this._net = new NetworkedStreams(this);
-    }
-
-    /** @internal */
-    onEnable(): void {
-        this._net?.enable();
-        //@ts-ignore
-        this._net?.addEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
-        //@ts-ignore
-        this._net?.addEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded);
-        this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
-        if (this.autoConnect) {
-            delay(1000).then(() => {
-                if (this.enabled && this.autoConnect && !this.isReceiving && !this.isSending && this.context.connection.isInRoom)
-                    this.share()
-                return 0;
-            });
-        }
-    }
-
-    /** @internal */
-    onDisable(): void {
-        //@ts-ignore
-        this._net?.removeEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
-        //@ts-ignore
-        this._net?.removeEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded);
-        this.context.connection.stopListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
-        this._net?.disable();
-        this.close();
-    }
-
-    private onJoinedRoom = async () => {
-        await delay(1000);
-        if (this.autoConnect && !this.isSending && !this.isReceiving && this.context.connection.isInRoom) {
-            this.share();
-        }
-    }
-
-    private _ensureVideoPlayer() {
-        const vp = new VideoPlayer();
-        vp.aspectMode = AspectMode.AdjustWidth;
-        GameObject.addComponent(this.gameObject, vp);
-        this._videoPlayer = vp;
-    }
-
-    private _activeShareRequest: Promise<void> | null = null;
-
-    /** Call to begin screensharing */
-    async share(opts?: ScreenCaptureOptions) {
-        if (this._activeShareRequest) return this._activeShareRequest;
-        this._activeShareRequest = this.internalShare(opts);
-        return this._activeShareRequest.then(() => {
-            return this._activeShareRequest = null;
-        })
-    }
-
-    private async internalShare(opts?: ScreenCaptureOptions) {
-        if (this.context.connection.isInRoom === false) {
-            console.warn("Can not start screensharing: requires network connection");
-            if (isDevEnvironment()) showBalloonWarning("Can not start screensharing: requires network connection. Add a SyncedRoom component or join a room first.");
-            return;
-        }
-
-        if (opts?.device)
-            this.device = opts.device;
-
-        if (!this.videoPlayer && this.requiresVideoPlayer) {
-            if (!this._videoPlayer) {
-                this._videoPlayer = GameObject.getComponent(this.gameObject, VideoPlayer) ?? undefined;
-            }
-            if (!this.videoPlayer) {
-                this._ensureVideoPlayer();
-            }
-            if (!this.videoPlayer) {
-                console.warn("Can not share video without a videoPlayer assigned");
-                return;
-            }
-        }
-
-        this._requestOpen = true;
-        try {
-
-            const settings: MediaTrackConstraints = opts?.constraints ?? {
-                echoCancellation: true,
-                autoGainControl: false,
-            };
-            const displayMediaOptions: MediaStreamConstraints = {
-                video: settings,
-                audio: settings,
-            };
-            const videoOptions = displayMediaOptions.video;
-            if (videoOptions !== undefined && typeof videoOptions !== "boolean") {
-                // Set default video settings
-                if (!videoOptions.width)
-                    videoOptions.width = { max: 1920 };
-                if (!videoOptions.height)
-                    videoOptions.height = { max: 1920 };
-                if (!videoOptions.aspectRatio)
-                    videoOptions.aspectRatio = { ideal: 1.7777777778 };
-                if (!videoOptions.frameRate)
-                    videoOptions.frameRate = { ideal: 24 };
-                if (!videoOptions.facingMode)
-                    videoOptions.facingMode = { ideal: "user" };
-            }
-
-            switch (this.device) {
-                // Capture a connected camera
-                case "Camera":
-                    this.tryShareUserCamera(displayMediaOptions, opts);
-                    break;
-
-                // capture any screen, will show a popup
-                case "Screen":
-                    {
-                        if (!navigator.mediaDevices.getDisplayMedia) {
-                            console.error("No getDisplayMedia support");
-                            return;
-                        }
-                        const myVideo = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
-                        if (this._requestOpen) {
-                            this.setStream(myVideo, ScreenCaptureMode.Sending);
-                        }
-                        else disposeStream(myVideo);
-                    }
-                    break;
-
-                // capture the canvas meaning the threejs view
-                case "Canvas":
-                    // looks like this doesnt work reliably on chrome https://stackoverflow.com/a/66848674
-                    // firefox updates fine
-                    // https://bugs.chromium.org/p/chromium/issues/detail?id=1156408
-                    const fps = 0;
-                    const stream = this.context.renderer.domElement.captureStream(fps);
-                    this.setStream(stream, ScreenCaptureMode.Sending);
-                    break;
-
-                case "Microphone":
-                    {
-                        if (!navigator.mediaDevices.getUserMedia) {
-                            console.error("No getDisplayMedia support");
-                            return;
-                        }
-                        displayMediaOptions.video = false;
-                        const myStream = await navigator.mediaDevices.getUserMedia(displayMediaOptions);
-                        if (this._requestOpen) {
-                            this.setStream(myStream, ScreenCaptureMode.Sending);
-                        }
-                        else disposeStream(myStream);
-                    }
-                    break
-
-                default:
-                    console.error("Can not start screen sharing: Unknown device type", this.device);
-
-            }
-        } catch (err: any) {
-            if (err.name === "NotAllowedError") {
-                // user cancelled stream selection
-                console.log("Selection cancelled");
-                this._requestOpen = false;
-                return;
-            }
-            console.error("Error opening video", err);
-        }
-    }
-
-    close() {
-        this._requestOpen = false;
-        if (this._currentStream) {
-            if (debug)
-                console.warn("Close current stream / disposing resources, stream was active?", this._currentStream.active);
-            this._net?.stopSendingStream(this._currentStream);
-            disposeStream(this._currentStream);
-            this._currentMode = ScreenCaptureMode.Idle;
-            this._currentStream = null;
-        }
-    }
-
-    private setStream(stream: MediaStream, mode: ScreenCaptureMode) {
-
-        if (stream === this._currentStream) return;
-
-        this.close();
-        if (!stream) return;
-
-        this._currentStream = stream;
-        this._requestOpen = true;
-        this._currentMode = mode;
-
-        const isVideoStream = this.device !== "Microphone";
-        const isSending = mode === ScreenCaptureMode.Sending;
-
-        if (isVideoStream) {
-            if (!this._videoPlayer)
-                this._ensureVideoPlayer();
-            if (this._videoPlayer)
-                this._videoPlayer.setVideo(stream);
-            else console.error("No video player assigned for video stream");
-        }
-        else {
-            if (!this._audioSource) {
-                this._audioSource = new AudioSource();
-                this._audioSource.spatialBlend = 0;
-                this._audioSource.volume = 1;
-                this.gameObject.addComponent(this._audioSource);
-            }
-            if (!isSending) {
-                if (debug) console.log("PLAY", stream.getAudioTracks())
-                this._audioSource.volume = 1;
-                this._audioSource?.play(stream);
-            }
-        }
-
-        if (isSending) {
-            this._net?.startSendingStream(stream);
-        }
-
-        // Mute audio for the video we are sending
-        if (isSending) {
-            if (this._videoPlayer)
-                this._videoPlayer.muted = true;
-            this._audioSource?.stop();
-        }
-
-        for (const track of stream.getTracks()) {
-            track.addEventListener("ended", () => {
-                if (debug) console.log("Track ended", track);
-                this.close();
-            });
-
-            if (debug) {
-                if (track.kind === "video") {
-                    if (isSending)
-                        console.log("Video →", track.getSettings());
-                    else
-                        console.log("Video ←", track.getSettings());
-                }
-            }
-        }
-
-    }
-
-    private onReceiveStream = (evt: StreamReceivedEvent) => {
-        if (evt.stream?.active !== true) return;
-        this.setStream(evt.stream, ScreenCaptureMode.Receiving);
-    }
-    private onCallEnded = (_evt: StreamEndedEvent) => {
-        if (debug) console.log("CALL ENDED", this.isReceiving, this?.screenspace)
-        if (this.isReceiving) this.screenspace = false;
-    }
-
-
-
-    private async tryShareUserCamera(constraints: MediaStreamConstraints, options?: ScreenCaptureOptions) {
-
-        // let newWindow = open('', 'example', 'width=300,height=300');
-        // if (window) {
-        //     newWindow!.document.body.innerHTML = "Please allow access to your camera and microphone";
-        // }
-
-        // TODO: allow user to select device
-        const devices = (await navigator.mediaDevices.enumerateDevices()).filter(d => d.kind === "videoinput");
-        if (debug)
-            console.log("Request camera. These are your kind:videoinput devices:\n", devices);
-
-        let foundDevice = false;
-
-        for (const dev of devices) {
-            try {
-                if (!this._requestOpen) {
-                    if (debug) console.log("Camera selection cancelled");
-                    break;
-                }
-
-                if (dev.kind !== "videoinput") {
-                    if (debug) console.log("Skipping non-video device", dev);
-                    continue;
-                }
-
-                const id = dev.deviceId;
-
-                // If the share method is called with filter options then those should be used
-                const hasOptionsFilter = options?.deviceId != undefined || options?.deviceFilter != undefined;
-                if (hasOptionsFilter) {
-                    if (options?.deviceId !== undefined) {
-                        if (id !== options.deviceId) {
-                            if (debug) console.log("Skipping device due to options.deviceId: " + dev.label + "; " + dev.deviceId);
-                            continue;
-                        }
-                    }
-                    if (options?.deviceFilter) {
-                        const useDevice = options.deviceFilter(dev);
-                        if (useDevice === false) {
-                            if (debug) console.log("Skipping device due to options.deviceFilter: " + dev.label + "; " + dev.deviceId);
-                            continue;
-                        }
-                    }
-                }
-                // If the share method was called without filter options then the component filter should be used
-                else if (this.deviceFilter) {
-                    const useDevice = this.deviceFilter(dev);
-                    if (useDevice === false) {
-                        if (debug) console.log("Skipping device due to ScreenShare.deviceFilter: " + dev.label + "; " + dev.deviceId);
-                        continue;
-                    }
-                    else if(debug)
-                        console.log("Selected device by filter", dev);
-                }
-                else if (this.deviceName) {
-                    const lowercaseLabel = dev.label.toLowerCase();
-                    const lowercaseName = this.deviceName.toLowerCase();
-                    const labelMatches = lowercaseLabel.includes(lowercaseName);
-                    const idMatches = dev.deviceId === this.deviceName;
-                    if (!labelMatches && !idMatches) {
-                        if (debug) console.log("Skipping device due to ScreenShare.deviceName: " + dev.label + "; " + dev.deviceId);
-                        continue;
-                    }
-                    else if(debug) console.log("Selected device by name", dev);
-                }
-
-                if (constraints.video !== false) {
-                    if (typeof constraints.video === "undefined" || typeof constraints.video === "boolean") {
-                        constraints.video = {};
-                    }
-                    constraints.video.deviceId = id;
-                }
-
-                foundDevice = true;
-                const userMedia = await navigator.mediaDevices.getUserMedia(constraints).catch(err => {
-                    console.error("Failed to get user media", err);
-                    return null;
-                })
-                if (userMedia === null) {
-                    continue;
-                }
-                else if (this._requestOpen) {
-                    this.setStream(userMedia, ScreenCaptureMode.Sending);
-                    if (debug)
-                        console.log("Selected camera", dev);
-                }
-                else {
-                    disposeStream(userMedia);
-                    if (debug)
-                        console.log("Camera selection cancelled");
-                }
-                break;
-            }
-            catch (err: any) {
-                // First message is firefox, second is chrome when the video source is already in use by another app
-                if (err.message === "Failed to allocate videosource" || err.message === "Could not start video source") {
-                    showBalloonWarning("Failed to start video: Try another camera (Code " + err.code + ")");
-                    console.warn(err);
-                    continue;
-                }
-                else {
-                    console.error("Failed to get user media", err.message, err.code, err);
-                }
-            }
-        }
-
-        if(!foundDevice && isDevEnvironment()){
-            showBalloonWarning("No camera found for sharing. Please connect a camera (see console for more information)");
-            console.warn("No camera found for sharing. Please connect a camera", devices, this.deviceName, "Using deviceFilter? " + this.deviceFilter != undefined, "Using options? " + options != undefined, "Using deviceName? " + this.deviceName != undefined, "Using options.deviceId? " + options?.deviceId != undefined, "Using options.deviceFilter? " + options?.deviceFilter != undefined);
-        }
-    }
-    // private _cameraSelectionWindow : Window | null = null;
-    // private openWindowToSelectCamera(){
-
-    // }
-}
-
+import { isDevEnvironment, showBalloonWarning } from "../engine/debug/index.js";
+import { RoomEvents } from "../engine/engine_networking.js";
+import { disposeStream, NetworkedStreamEvents, NetworkedStreams, PeerHandle, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js";
+import { serializable } from "../engine/engine_serialization.js";
+import { delay, getParam } from "../engine/engine_utils.js";
+import { AudioSource } from "./AudioSource.js";
+import { Behaviour, GameObject } from "./Component.js";
+import type { IPointerClickHandler, PointerEventData } from "./ui/PointerEvents.js";
+import { AspectMode, VideoPlayer } from "./VideoPlayer.js";
+
+const debug = getParam("debugscreensharing");
+
+/**
+ * ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room.
+ */
+export enum ScreenCaptureDevice {
+    /**
+     * Capture the screen of the user.
+     */
+    Screen = 0,
+    /**
+     * Capture the camera of the user.
+     */
+    Camera = 1,
+    /** Please note that canvas streaming might not work reliably on chrome: https://bugs.chromium.org/p/chromium/issues/detail?id=1156408 */
+    Canvas = 2,
+    /** When using Microphone only the voice will be send */
+    Microphone = 3
+}
+
+/**
+ * {@link ScreenCapture} allows you to share your screen, camera or microphone with other users in the networked room.
+ */
+declare type ScreenCaptureDeviceTypes = keyof typeof ScreenCaptureDevice;
+
+/**
+ * The current mode of the {@link ScreenCapture} component.
+ */
+export enum ScreenCaptureMode {
+    Idle = 0,
+    Sending = 1,
+    Receiving = 2
+}
+
+/**
+ * Options for the {@link ScreenCapture} component when starting to share a stream by calling the {@link ScreenCapture.share}.
+ */
+export declare type ScreenCaptureOptions = {
+    /**
+     * You can specify the device type to capture (e.g. Screen, Camera, Microphone)
+     */
+    device?: ScreenCaptureDeviceTypes,
+    /**
+     * Constraints for the media stream like resolution, frame rate, etc.
+     * @see https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamConstraints
+     */
+    constraints?: MediaTrackConstraints,
+    /** Filter video device by id. Alternatively pass in a deviceFilter callback to manually filter available devices */
+    deviceId?: string,
+    /** Return false to skip the available device */
+    deviceFilter?: (device: MediaDeviceInfo) => boolean,
+}
+
+/** 
+ * The ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room.   
+ * When the stream is active the video will be displayed on the VideoPlayer component attached to the same GameObject.  
+ * 
+ * Note: For debugging append `?debugscreensharing` to the URL to see more information in the console.
+ * 
+ * By default the component will start sharing the screen when the user clicks on the object this component is attached to. You can set {@link device} This behaviour can be disabled by setting `allowStartOnClick` to false.  
+ * It is also possible to start the stream manually from your code by calling the {@link share} method.
+ * 
+ * @category Networking
+ * @group Components
+*/
+export class ScreenCapture extends Behaviour implements IPointerClickHandler {
+
+    /**
+     * When enabled the stream will start when the user clicks on the object this component is attached to  
+     * It is also possible to start the stream manually from your code by calling the {@link share} method    
+     * To modify what type of device is shared you can set the {@link device} property.
+     * @default true
+     */
+    @serializable()
+    allowStartOnClick: boolean = true;
+
+    /** @internal */
+    onPointerEnter() {
+        if (this.context.connection.allowEditing == false) return;
+        if (!this.allowStartOnClick) return;
+        this.context.input.setCursor("pointer");
+    }
+    /** @internal */
+    onPointerExit() {
+        if (this.context.connection.allowEditing == false) return;
+        if (!this.allowStartOnClick) return;
+        this.context.input.unsetCursor("pointer");
+    }
+
+    /** @internal */
+    onPointerClick(evt: PointerEventData) {
+        if (this.context.connection.allowEditing == false) return;
+        if (!this.allowStartOnClick) return;
+        if (evt && evt.pointerId !== 0) return;
+        if (this.isReceiving && this.videoPlayer?.isPlaying) {
+            if (this.videoPlayer)
+                this.videoPlayer.screenspace = !this.videoPlayer.screenspace;
+            return;
+
+        }
+        if (this.isSending) {
+            this.close();
+            return;
+        }
+        this.share();
+    }
+
+
+    /** When enabled the stream will start when this component becomes active (enabled in the scene) */
+    @serializable()
+    autoConnect: boolean = false;
+
+    /**
+     * If a VideoPlayer component is assigned to this property the video will be displayed on the VideoPlayer component.
+     */
+    @serializable(VideoPlayer)
+    set videoPlayer(val: VideoPlayer | undefined) {
+        if (this._videoPlayer && (this.isSending || this.isReceiving)) {
+            this._videoPlayer.stop();
+        }
+        this._videoPlayer = val;
+        if (this._videoPlayer && this._currentStream && (this.isSending || this.isReceiving)) {
+            this._videoPlayer.setVideo(this._currentStream);
+        }
+    }
+    get videoPlayer() { return this._videoPlayer; }
+    private _videoPlayer?: VideoPlayer;
+    private _audioSource?: AudioSource;
+
+    /**
+     * When enabled the video will be displayed in the screenspace of the VideoPlayer component.
+     */
+    get screenspace() { return this.videoPlayer?.screenspace ?? false; }
+    set screenspace(v: boolean) { if (this.videoPlayer) this.videoPlayer.screenspace = v; }
+
+    /**
+     * Which streaming device type should be used when starting to share (if {@link share} is called without a device option). Options are Screen, Camera, Microphone.  
+     * This is e.g. used if `allowStartOnClick` is enabled and the user clicks on the object. 
+     * @default Screen 
+     */
+    @serializable()
+    device: ScreenCaptureDeviceTypes = "Screen";
+
+    /**
+     * If assigned the device the device will be selected by this id or label when starting to share.  
+     * Note: This is only supported for `Camera` devices
+     */
+    @serializable()
+    deviceName?: string;
+
+    /**
+     * Filter which device should be chosen for sharing by id or label.  
+     * Assign a method to this property to manually filter the available devices.  
+     */
+    deviceFilter?: (device: MediaDeviceInfo) => boolean;
+
+    /**
+     * the current stream that is being shared or received  
+     * @link https://developer.mozilla.org/en-US/docs/Web/API/MediaStream
+     */
+    get currentScream(): MediaStream | null {
+        return this._currentStream;
+    }
+    get currentMode(): ScreenCaptureMode {
+        return this._currentMode;
+    }
+
+    /**
+     * @returns true if the component is currently sending a stream
+     */
+    get isSending() {
+        return this._currentStream?.active && this._currentMode === ScreenCaptureMode.Sending;
+    }
+    /**
+     * @returns true if the component is currently receiving a stream
+     */
+    get isReceiving() {
+        if (this._currentMode === ScreenCaptureMode.Receiving) {
+            if (!this._currentStream || this._currentStream.active === false) return false;
+            // if any track is still live consider it active
+            const tracks = this._currentStream.getTracks();
+            for (const track of tracks) {
+                if (track.readyState === "live") return true;
+            }
+        }
+        return false;
+    }
+
+    private get requiresVideoPlayer() { 
+        return this.device !== "Microphone";
+    }
+    private _net?: NetworkedStreams;
+    private _requestOpen: boolean = false;
+    private _currentStream: MediaStream | null = null;
+    private _currentMode: ScreenCaptureMode = ScreenCaptureMode.Idle;
+
+    /** @internal */
+    awake() {
+        // Resolve the device type if it is a number
+        if (typeof this.device === "number") {
+            this.device = ScreenCaptureDevice[this.device] as ScreenCaptureDeviceTypes;
+        }
+
+        if (debug)
+            console.log("Screensharing", this.name, this);
+        AudioSource.registerWaitForAllowAudio(() => {
+            if (this._videoPlayer && this._currentStream && this._currentMode === ScreenCaptureMode.Receiving) {
+                this._videoPlayer.playInBackground = true;
+                this._videoPlayer.setVideo(this._currentStream);
+            }
+        });
+        this._net = new NetworkedStreams(this);
+    }
+
+    /** @internal */
+    onEnable(): void {
+        this._net?.enable();
+        //@ts-ignore
+        this._net?.addEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
+        //@ts-ignore
+        this._net?.addEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded);
+        this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
+        if (this.autoConnect) {
+            delay(1000).then(() => {
+                if (this.enabled && this.autoConnect && !this.isReceiving && !this.isSending && this.context.connection.isInRoom)
+                    this.share()
+                return 0;
+            });
+        }
+    }
+
+    /** @internal */
+    onDisable(): void {
+        //@ts-ignore
+        this._net?.removeEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
+        //@ts-ignore
+        this._net?.removeEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded);
+        this.context.connection.stopListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
+        this._net?.disable();
+        this.close();
+    }
+
+    private onJoinedRoom = async () => {
+        await delay(1000);
+        if (this.autoConnect && !this.isSending && !this.isReceiving && this.context.connection.isInRoom) {
+            this.share();
+        }
+    }
+
+    private _ensureVideoPlayer() {
+        const vp = new VideoPlayer();
+        vp.aspectMode = AspectMode.AdjustWidth;
+        GameObject.addComponent(this.gameObject, vp);
+        this._videoPlayer = vp;
+    }
+
+    private _activeShareRequest: Promise<void> | null = null;
+
+    /** Call to begin screensharing */
+    async share(opts?: ScreenCaptureOptions) {
+        if (this._activeShareRequest) return this._activeShareRequest;
+        this._activeShareRequest = this.internalShare(opts);
+        return this._activeShareRequest.then(() => {
+            return this._activeShareRequest = null;
+        })
+    }
+
+    private async internalShare(opts?: ScreenCaptureOptions) {
+        if (this.context.connection.isInRoom === false) {
+            console.warn("Can not start screensharing: requires network connection");
+            if (isDevEnvironment()) showBalloonWarning("Can not start screensharing: requires network connection. Add a SyncedRoom component or join a room first.");
+            return;
+        }
+
+        if (opts?.device)
+            this.device = opts.device;
+
+        if (!this.videoPlayer && this.requiresVideoPlayer) {
+            if (!this._videoPlayer) {
+                this._videoPlayer = GameObject.getComponent(this.gameObject, VideoPlayer) ?? undefined;
+            }
+            if (!this.videoPlayer) {
+                this._ensureVideoPlayer();
+            }
+            if (!this.videoPlayer) {
+                console.warn("Can not share video without a videoPlayer assigned");
+                return;
+            }
+        }
+
+        this._requestOpen = true;
+        try {
+
+            const settings: MediaTrackConstraints = opts?.constraints ?? {
+                echoCancellation: true,
+                autoGainControl: false,
+            };
+            const displayMediaOptions: MediaStreamConstraints = {
+                video: settings,
+                audio: settings,
+            };
+            const videoOptions = displayMediaOptions.video;
+            if (videoOptions !== undefined && typeof videoOptions !== "boolean") {
+                // Set default video settings
+                if (!videoOptions.width)
+                    videoOptions.width = { max: 1920 };
+                if (!videoOptions.height)
+                    videoOptions.height = { max: 1920 };
+                if (!videoOptions.aspectRatio)
+                    videoOptions.aspectRatio = { ideal: 1.7777777778 };
+                if (!videoOptions.frameRate)
+                    videoOptions.frameRate = { ideal: 24 };
+                if (!videoOptions.facingMode)
+                    videoOptions.facingMode = { ideal: "user" };
+            }
+
+            switch (this.device) {
+                // Capture a connected camera
+                case "Camera":
+                    this.tryShareUserCamera(displayMediaOptions, opts);
+                    break;
+
+                // capture any screen, will show a popup
+                case "Screen":
+                    {
+                        if (!navigator.mediaDevices.getDisplayMedia) {
+                            console.error("No getDisplayMedia support");
+                            return;
+                        }
+                        const myVideo = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+                        if (this._requestOpen) {
+                            this.setStream(myVideo, ScreenCaptureMode.Sending);
+                        }
+                        else disposeStream(myVideo);
+                    }
+                    break;
+
+                // capture the canvas meaning the threejs view
+                case "Canvas":
+                    // looks like this doesnt work reliably on chrome https://stackoverflow.com/a/66848674
+                    // firefox updates fine
+                    // https://bugs.chromium.org/p/chromium/issues/detail?id=1156408
+                    const fps = 0;
+                    const stream = this.context.renderer.domElement.captureStream(fps);
+                    this.setStream(stream, ScreenCaptureMode.Sending);
+                    break;
+
+                case "Microphone":
+                    {
+                        if (!navigator.mediaDevices.getUserMedia) {
+                            console.error("No getDisplayMedia support");
+                            return;
+                        }
+                        displayMediaOptions.video = false;
+                        const myStream = await navigator.mediaDevices.getUserMedia(displayMediaOptions);
+                        if (this._requestOpen) {
+                            this.setStream(myStream, ScreenCaptureMode.Sending);
+                        }
+                        else disposeStream(myStream);
+                    }
+                    break
+
+                default:
+                    console.error("Can not start screen sharing: Unknown device type", this.device);
+
+            }
+        } catch (err: any) {
+            if (err.name === "NotAllowedError") {
+                // user cancelled stream selection
+                console.log("Selection cancelled");
+                this._requestOpen = false;
+                return;
+            }
+            console.error("Error opening video", err);
+        }
+    }
+
+    close() {
+        this._requestOpen = false;
+        if (this._currentStream) {
+            if (debug)
+                console.warn("Close current stream / disposing resources, stream was active?", this._currentStream.active);
+            this._net?.stopSendingStream(this._currentStream);
+            disposeStream(this._currentStream);
+            this._currentMode = ScreenCaptureMode.Idle;
+            this._currentStream = null;
+        }
+    }
+
+    private setStream(stream: MediaStream, mode: ScreenCaptureMode) {
+
+        if (stream === this._currentStream) return;
+
+        this.close();
+        if (!stream) return;
+
+        this._currentStream = stream;
+        this._requestOpen = true;
+        this._currentMode = mode;
+
+        const isVideoStream = this.device !== "Microphone";
+        const isSending = mode === ScreenCaptureMode.Sending;
+
+        if (isVideoStream) {
+            if (!this._videoPlayer)
+                this._ensureVideoPlayer();
+            if (this._videoPlayer)
+                this._videoPlayer.setVideo(stream);
+            else console.error("No video player assigned for video stream");
+        }
+        else {
+            if (!this._audioSource) {
+                this._audioSource = new AudioSource();
+                this._audioSource.spatialBlend = 0;
+                this._audioSource.volume = 1;
+                this.gameObject.addComponent(this._audioSource);
+            }
+            if (!isSending) {
+                if (debug) console.log("PLAY", stream.getAudioTracks())
+                this._audioSource.volume = 1;
+                this._audioSource?.play(stream);
+            }
+        }
+
+        if (isSending) {
+            this._net?.startSendingStream(stream);
+        }
+
+        // Mute audio for the video we are sending
+        if (isSending) {
+            if (this._videoPlayer)
+                this._videoPlayer.muted = true;
+            this._audioSource?.stop();
+        }
+
+        for (const track of stream.getTracks()) {
+            track.addEventListener("ended", () => {
+                if (debug) console.log("Track ended", track);
+                this.close();
+            });
+
+            if (debug) {
+                if (track.kind === "video") {
+                    if (isSending)
+                        console.log("Video →", track.getSettings());
+                    else
+                        console.log("Video ←", track.getSettings());
+                }
+            }
+        }
+
+    }
+
+    private onReceiveStream = (evt: StreamReceivedEvent) => {
+        if (evt.stream?.active !== true) return;
+        this.setStream(evt.stream, ScreenCaptureMode.Receiving);
+    }
+    private onCallEnded = (_evt: StreamEndedEvent) => {
+        if (debug) console.log("CALL ENDED", this.isReceiving, this?.screenspace)
+        if (this.isReceiving) this.screenspace = false;
+    }
+
+
+
+    private async tryShareUserCamera(constraints: MediaStreamConstraints, options?: ScreenCaptureOptions) {
+
+        // let newWindow = open('', 'example', 'width=300,height=300');
+        // if (window) {
+        //     newWindow!.document.body.innerHTML = "Please allow access to your camera and microphone";
+        // }
+
+        // TODO: allow user to select device
+        const devices = (await navigator.mediaDevices.enumerateDevices()).filter(d => d.kind === "videoinput");
+        if (debug)
+            console.log("Request camera. These are your kind:videoinput devices:\n", devices);
+
+        let foundDevice = false;
+
+        for (const dev of devices) {
+            try {
+                if (!this._requestOpen) {
+                    if (debug) console.log("Camera selection cancelled");
+                    break;
+                }
+
+                if (dev.kind !== "videoinput") {
+                    if (debug) console.log("Skipping non-video device", dev);
+                    continue;
+                }
+
+                const id = dev.deviceId;
+
+                // If the share method is called with filter options then those should be used
+                const hasOptionsFilter = options?.deviceId != undefined || options?.deviceFilter != undefined;
+                if (hasOptionsFilter) {
+                    if (options?.deviceId !== undefined) {
+                        if (id !== options.deviceId) {
+                            if (debug) console.log("Skipping device due to options.deviceId: " + dev.label + "; " + dev.deviceId);
+                            continue;
+                        }
+                    }
+                    if (options?.deviceFilter) {
+                        const useDevice = options.deviceFilter(dev);
+                        if (useDevice === false) {
+                            if (debug) console.log("Skipping device due to options.deviceFilter: " + dev.label + "; " + dev.deviceId);
+                            continue;
+                        }
+                    }
+                }
+                // If the share method was called without filter options then the component filter should be used
+                else if (this.deviceFilter) {
+                    const useDevice = this.deviceFilter(dev);
+                    if (useDevice === false) {
+                        if (debug) console.log("Skipping device due to ScreenShare.deviceFilter: " + dev.label + "; " + dev.deviceId);
+                        continue;
+                    }
+                    else if(debug)
+                        console.log("Selected device by filter", dev);
+                }
+                else if (this.deviceName) {
+                    const lowercaseLabel = dev.label.toLowerCase();
+                    const lowercaseName = this.deviceName.toLowerCase();
+                    const labelMatches = lowercaseLabel.includes(lowercaseName);
+                    const idMatches = dev.deviceId === this.deviceName;
+                    if (!labelMatches && !idMatches) {
+                        if (debug) console.log("Skipping device due to ScreenShare.deviceName: " + dev.label + "; " + dev.deviceId);
+                        continue;
+                    }
+                    else if(debug) console.log("Selected device by name", dev);
+                }
+
+                if (constraints.video !== false) {
+                    if (typeof constraints.video === "undefined" || typeof constraints.video === "boolean") {
+                        constraints.video = {};
+                    }
+                    constraints.video.deviceId = id;
+                }
+
+                foundDevice = true;
+                const userMedia = await navigator.mediaDevices.getUserMedia(constraints).catch(err => {
+                    console.error("Failed to get user media", err);
+                    return null;
+                })
+                if (userMedia === null) {
+                    continue;
+                }
+                else if (this._requestOpen) {
+                    this.setStream(userMedia, ScreenCaptureMode.Sending);
+                    if (debug)
+                        console.log("Selected camera", dev);
+                }
+                else {
+                    disposeStream(userMedia);
+                    if (debug)
+                        console.log("Camera selection cancelled");
+                }
+                break;
+            }
+            catch (err: any) {
+                // First message is firefox, second is chrome when the video source is already in use by another app
+                if (err.message === "Failed to allocate videosource" || err.message === "Could not start video source") {
+                    showBalloonWarning("Failed to start video: Try another camera (Code " + err.code + ")");
+                    console.warn(err);
+                    continue;
+                }
+                else {
+                    console.error("Failed to get user media", err.message, err.code, err);
+                }
+            }
+        }
+
+        if(!foundDevice && isDevEnvironment()){
+            showBalloonWarning("No camera found for sharing. Please connect a camera (see console for more information)");
+            console.warn("No camera found for sharing. Please connect a camera", devices, this.deviceName, "Using deviceFilter? " + this.deviceFilter != undefined, "Using options? " + options != undefined, "Using deviceName? " + this.deviceName != undefined, "Using options.deviceId? " + options?.deviceId != undefined, "Using options.deviceFilter? " + options?.deviceFilter != undefined);
+        }
+    }
+    // private _cameraSelectionWindow : Window | null = null;
+    // private openWindowToSelectCamera(){
+
+    // }
+}
+