@needle-tools/engine 2.30.0-pre → 2.31.0-pre

package/src/engine-components/ScreenCapture.ts

@@ -6,75 +6,269 @@ import { RoomEvents } from "../engine/engine_networking";
 import { UserJoinedOrLeftRoomModel } from "../engine/engine_networking";
 import { serializeable } from "../engine/engine_serialization";
 import { IModel } from "../engine/engine_networking";
+import { IPointerClickHandler } from "./ui/PointerEvents";
+import { EventDispatcher } from "three";
+import { AudioSource } from "./AudioSource";
+import { getWorldScale, setWorldScale } from "../engine/engine_three_utils";
+
+
+export enum ScreenCaptureDevice {
+    Screen = 0,
+    Camera = 1
+}
+
+export enum ScreenCaptureMode {
+    Idle = 0,
+    Sending = 1,
+    Receiving = 2
+}
+
+export enum AspectMode {
+    None = 0,
+    AdjustHeight = 1,
+    AdjustWidth = 2,
+}
+
+function disposeStream(str: MediaStream | null | undefined) {
+    if (!str) return;
+    for (const cap of str.getTracks())
+        cap.stop();
+}
+
+export class ScreenCapture extends Behaviour implements IPointerClickHandler {
+
+    onPointerClick() {
+        if (this.isSending) {
+            this.close();
+            return;
+        }
+        this.share();
+    }
+
+
 
-export class ScreenCapture extends Behaviour {
 
     @serializeable()
-    streamOnAwake: boolean = true;
+    device: ScreenCaptureDevice = ScreenCaptureDevice.Screen;
 
-    start() {
-        console.warn("EXPERIMENTAL screen capture", this);
-        if (this.streamOnAwake)
-            this.open();
+    @serializeable()
+    aspectMode: AspectMode = AspectMode.AdjustHeight;
+
+
+    get isSending() {
+        return this._currentStream && this._currentMode === ScreenCaptureMode.Sending;
+    }
+    get isReceiving() {
+        return this._currentStream && this._currentMode === ScreenCaptureMode.Receiving;
     }
 
-    private requestOpen: boolean = false;
-    private captureStream: MediaStream | null = null;
+    private _net?: NetworkedVideo;
+    private _video?: VideoPlayer;
+    private _requestOpen: boolean = false;
+    private _currentStream: MediaStream | null = null;
+    private _currentMode: ScreenCaptureMode = ScreenCaptureMode.Idle;
 
-    async open(force: boolean = false) {
-        if (this.captureStream && !force) return;
-        this.close();
-        this.requestOpen = true;
+    awake() {
+        AudioSource.registerWaitForAllowAudio(() => {
+            if (this._video && this._currentStream && this._currentMode === ScreenCaptureMode.Receiving) {
+                this._video.setVideo(this._currentStream);
+            }
+        });
+    }
+
+    start() {
+        this._video = GameObject.getComponentInChildren(this.gameObject, VideoPlayer) ?? undefined;
+        if (!this._video) {
+            console.error("Missing VideoPlayer component");
+            return;
+        }
+        const handle = PeerHandle.getOrCreate(this.context, this.guid);
+        this._net = new NetworkedVideo(this.context, handle);
+        this._net.enable();
+        //@ts-ignore
+        this._net.addEventListener(PeerEvent.ReceiveVideo, this.onReceiveVideo.bind(this));
+    }
+
+    async share() {
+        this._requestOpen = true;
         try {
-            const player = GameObject.getComponentInChildren(this.gameObject, VideoPlayer);
-            if (player) {
+            if (this._video) {
+
+                const settings: MediaTrackConstraints = {
+                    echoCancellation: true,
+                    autoGainControl: false,
+                };
                 const displayMediaOptions: DisplayMediaStreamConstraints = {
-                    video: true,
-                    audio: true
+                    video: settings,
+                    audio: settings,
                 };
-                this.captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
-                if (this.captureStream && this.requestOpen) {
-                    //@ts-ignore
-                    player.clip = this.captureStream;
-                    player.create(true);
+
+                switch (this.device) {
+                    case ScreenCaptureDevice.Camera:
+                        this.tryShareUserCamera(displayMediaOptions);
+                        break;
+
+                    case ScreenCaptureDevice.Screen:
+                        if (!navigator.mediaDevices.getDisplayMedia) {
+                            console.error("No getDisplayMedia support");
+                            return;
+                        }
+                        const myVideo = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+                        if (this._requestOpen) {
+                            this.setVideo(myVideo, ScreenCaptureMode.Sending);
+                        }
+                        else disposeStream(myVideo);
+                        break;
                 }
-                this.handleNetworkedVideo();
             }
-        } catch (err) {
-            console.error("Error: " + err);
+        } catch (err: any) {
+            if (err.name === "NotAllowedError") {
+                // user cancelled stream selection
+                console.log("Selection cancelled");
+                this._requestOpen = false;
+                return;
+            }
+            console.error("Error opening video", err);
         }
     }
 
     close() {
-        this.requestOpen = false;
-        if (this.captureStream) {
-            for (const cap of this.captureStream.getTracks())
-                cap.stop();
-            this.captureStream = null;
+        this._requestOpen = false;
+        if (this._currentStream) {
+            console.warn("Close current stream / disposing resources");
+            disposeStream(this._currentStream);
+            this._net?.stopSendingVideo(this._currentStream);
+            this._currentMode = ScreenCaptureMode.Idle;
+            this._currentStream = null;
         }
     }
 
-    private handleNetworkedVideo() {
-        if (!this.context.connection.isConnected) {
-            console.warn("Will not stream video - not connected to room");
-            return;
-        }
-        if (this.captureStream) {
-            const handle = PeerHandle.getOrCreate(this.context, this.guid);
-            handle.enable();
-            // if (!this.peer)
-            //     this.peer = new Peer();
-            const net = new NetworkedVideo(this.context, handle);
-            net.enable();
-            net.startSendingVideo(this.captureStream);
+    private setVideo(stream: MediaStream, mode: ScreenCaptureMode) {
+        if (stream === this._currentStream) return;
+        this.close();
+        if (!stream || !this._video) return;
+        this._currentStream = stream;
+        this._requestOpen = true;
+        this._currentMode = mode;
+        this._video.setVideo(stream);
+        if (mode === ScreenCaptureMode.Sending)
+            this._net?.startSendingVideo(stream);
+
+        this._targetObjects = undefined;
+        this.updateAspect();
+    }
+
+    private onReceiveVideo(evt: ReceiveVideoEvent) {
+        this.setVideo(evt.stream, ScreenCaptureMode.Receiving);
+    }
+
+
+
+
+    private async tryShareUserCamera(opts: DisplayMediaStreamConstraints) {
+
+        // let newWindow = open('', 'example', 'width=300,height=300');
+        // if (window) {
+        //     newWindow!.document.body.innerHTML = "Please allow access to your camera and microphone";
+        // }
+
+        // TODO: allow user to select device
+        const devices = (await navigator.mediaDevices.enumerateDevices()).filter(d => d.kind === "videoinput");
+        console.log("Request camera", devices);
+        for (const dev of devices) {
+            try {
+                if (!this._requestOpen) break;
+                if (dev.kind !== "videoinput") continue;
+                const id = dev.deviceId;
+                if (opts.video !== false) {
+                    if (typeof opts.video === "undefined" || typeof opts.video === "boolean") {
+                        opts.video = {};
+                    }
+                    opts.video.deviceId = id;
+                }
+                const userMedia = await navigator.mediaDevices.getUserMedia(opts);
+                if (this._requestOpen) {
+                    this.setVideo(userMedia, ScreenCaptureMode.Sending);
+                }
+                else disposeStream(userMedia);
+                console.log("Selected camera", dev);
+                break;
+            }
+            catch (err) {
+                console.warn(err);
+            }
         }
     }
+    // private _cameraSelectionWindow : Window | null = null;
+    // private openWindowToSelectCamera(){
+
+    // }
+
+    
+
+
+    private _targetObjects?: Array<THREE.Object3D>;
+
+    private updateAspect() {
+        this.startCoroutine(this.updateAspectImpl());
+    }
 
+    private _updateAspectRoutineId: number = -1;
+    private *updateAspectImpl() {
+        const id = ++this._updateAspectRoutineId;
+        while (this.aspectMode !== AspectMode.None && this._currentStream && id === this._updateAspectRoutineId) {
+            const video = this._video;
+            const stream = this._currentStream;
+            if (!video) return;
+            if (!stream) return;
+            let aspect: number | undefined = undefined;
+            for (const track of stream.getVideoTracks()) {
+                const settings = track.getSettings();
+                if (settings.width && settings.height) {
+                    aspect = settings.width / settings.height;
+                    break;
+                }
+            }
+            if (aspect === undefined) return;
+            if (!this._targetObjects)
+                this._targetObjects = video.getTargetObjects();
+            for (const obj of this._targetObjects) {
+                let worldAspect = 1;
+                if (obj.parent) {
+                    const parentScale = getWorldScale(obj.parent);
+                    worldAspect = parentScale.x / parentScale.y;
+                }
+                switch (this.aspectMode) {
+                    case AspectMode.AdjustHeight:
+                        obj.scale.y = 1 / aspect * obj.scale.x * worldAspect;
+                        break;
+                    case AspectMode.AdjustWidth:
+                        obj.scale.x = aspect * obj.scale.y * worldAspect;
+                        break;
+                }
+            }
+            yield;
+        }
+    }
 }
 
 
-enum PeerHandleEvent {
+/////// PEER
+
+enum PeerEvent {
     Connected = "peer-user-connected",
+    ReceiveVideo = "receive-video",
+    Disconnected = "peer-user-disconnected",
+}
+
+class ReceiveVideoEvent {
+    readonly type = PeerEvent.ReceiveVideo;
+    readonly stream: MediaStream;
+    readonly target: CallHandle;
+    constructor(stream: MediaStream, target: CallHandle) {
+        this.stream = stream
+        this.target = target;
+    }
 }
 
 class PeerUserConnectedModel implements IModel {
@@ -94,22 +288,48 @@ enum CallDirection {
     Outgoing = "outgoing",
 }
 
-class CallHandle {
+class CallHandle extends EventDispatcher {
+    readonly userId: string;
     readonly direction: CallDirection;
     readonly call: Peer.MediaConnection;
-    stream: MediaStream | null;
+    get stream() { return this._stream; };
+
+    private _stream: MediaStream | null = null;
+    private _isDisposed: boolean = false;
+
+    close() {
+        if (this._isDisposed) return;
+        this._isDisposed = true;
+        this.call.close();
+        disposeStream(this._stream);
+    }
+
+    get isOpen() {
+        return this.call.peerConnection?.connectionState === "connected";// && this._stream?.active;
+    }
 
-    constructor(call: Peer.MediaConnection, direction: CallDirection) {
+    get isClosed() {
+        return !this.isOpen;
+    }
+
+    constructor(userId: string, call: Peer.MediaConnection, direction: CallDirection) {
+        super();
+        this.userId = userId;
         this.call = call;
         this.direction = direction;
-        this.stream = null;
+        this._stream = null;
         call.on("stream", stream => {
-            this.stream = stream;
+            console.log("Receive video", stream.getAudioTracks(), stream.getVideoTracks());
+            this._stream = stream;
+            if (direction === CallDirection.Incoming) {
+                const args: ReceiveVideoEvent = new ReceiveVideoEvent(stream, this);
+                this.dispatchEvent(args);
+            }
         });
     }
 }
 
-class PeerHandle {
+class PeerHandle extends EventDispatcher {
 
     private static readonly instances: Map<string, PeerHandle> = new Map();
 
@@ -125,6 +345,12 @@ class PeerHandle {
         return peer;
     }
 
+    getMyPeerId(): string | undefined {
+        if (this.context.connection.connectionId)
+            return this.getPeerIdFromUserId(this.context.connection.connectionId);
+        return undefined;
+    }
+
     getPeerIdFromUserId(userConnectionId: string): string {
         // we build the peer id ourselves so we dont need to wait for peer to report it
         return this.id + "-" + userConnectionId;
@@ -135,13 +361,8 @@ class PeerHandle {
     }
 
     makeCall(peerId: string, stream: MediaStream): CallHandle | undefined {
-        // if (userId === this.context.connection.connectionId) {
-        //     console.warn("Can not call self");
-        //     return;
-        // }
-        // const peerId = this.getUserPeerId(userId);
-        console.log("CALL", peerId);
-        const call = this._peer?.call(peerId, stream);
+        const opts = { metadata: { userId: this.context.connection.connectionId } };
+        const call = this._peer?.call(peerId, stream, opts);
         if (call)
             return this.registerCall(call, CallDirection.Outgoing);
         return undefined;
@@ -156,6 +377,7 @@ class PeerHandle {
     private _outgoingCalls: CallHandle[] = [];
 
     private constructor(context: Context, id: string) {
+        super();
         this.context = context;
         this.id = id;
         this.setupPeer();
@@ -203,7 +425,7 @@ class PeerHandle {
         if (this._enabledPeer) return;
         this._enabledPeer = true;
         if (!this._peer) {
-            const peerId = this.getPeerIdFromUserId(this.context.connection.connectionId);
+            const peerId = this.getMyPeerId();
             this._peer = new Peer(peerId);
         }
         if (this._enabled)
@@ -218,31 +440,60 @@ class PeerHandle {
     }
 
     private unsubscribePeerEvents() {
+        // TODO: unsubscribe
     }
 
     private onPeerConnect(id): void {
-        console.log("connect", id);
-        this.context.connection.send(PeerHandleEvent.Connected, new PeerUserConnectedModel(this, id));
+        console.log("Peer connected as", id);
+        this.context.connection.send(PeerEvent.Connected, new PeerUserConnectedModel(this, id));
     }
 
     private onPeerReceivingCall(call: Peer.MediaConnection): void {
-        console.log("RECEIVE CALL");
         call.answer();
         this.registerCall(call, CallDirection.Incoming);
     }
 
     private registerCall(call: Peer.MediaConnection, direction: CallDirection): CallHandle {
+
+        const meta = call.metadata;
+        if (!meta || !meta.userId) {
+            console.error("Missing call metadata", call);
+        }
+        const userId = meta.userId;
+
+        if (direction === CallDirection.Incoming) console.log("Receive call from", call.metadata);
+        else console.log("Make call to", call.metadata);
+
         const arr = direction === CallDirection.Incoming ? this._incomingCalls : this._outgoingCalls;
-        const handle = new CallHandle(call, direction);
+        const handle = new CallHandle(userId, call, direction);
         arr.push(handle);
         call.on("error", err => {
-            console.error(err);
+            console.error("Call error", err);
         });
         call.on("close", () => {
+            console.log("Call ended", call.metadata);
+            call.close();
             const index = arr.indexOf(handle);
             if (index !== -1)
                 arr.splice(index, 1);
         });
+
+        if (direction === CallDirection.Incoming) {
+
+            handle.addEventListener(PeerEvent.ReceiveVideo, e => {
+                this.dispatchEvent(e);
+            });
+
+            call.on("stream", () => {
+                // workaround for https://github.com/peers/peerjs/issues/636
+                let closeInterval = setInterval(() => {
+                    if (!handle.isOpen) {
+                        clearInterval(closeInterval);
+                        handle.close();
+                    }
+                }, 2000);
+            });
+        }
         return handle;
     }
 
@@ -252,36 +503,29 @@ class PeerHandle {
 }
 
 
-type UserVideoCall = {
-    call: Peer.MediaConnection;
-    stream: MediaStream;
-    userId: string;
-}
+// type UserVideoCall = {
+//     call: Peer.MediaConnection;
+//     stream: MediaStream;
+//     userId: string;
+// }
 
-class NetworkedVideo {
+// type IncomingStreamArgs = {
+//     stream: MediaStream;
+//     userId: string;
+// }
+
+class NetworkedVideo extends EventDispatcher {
 
     private readonly context: Context;
     private readonly peer: PeerHandle;
 
-    private _sendingVideoStreams: Map<MediaStream, UserVideoCall[]> = new Map();
+    // private _receiveVideoStreamListeners: Array<(info: IncomingStreamArgs) => void> = [];
+    private _sendingVideoStreams: Map<MediaStream, CallHandle[]> = new Map();
 
     constructor(context: Context, peer: PeerHandle) {
+        super();
         this.context = context;
         this.peer = peer;
-
-        // this.peer.on("open", (id) => {
-        //     console.log("PEERID", id);
-        // });
-        // // this.peer.on("call", (call) => {
-        // // });
-        // // this.peer.on("connection", (conn) => {
-        // // });
-        // // this.peer.on("error", (err) => {
-        // // });
-        // // this.peer.on("disconnected", () => {
-        // // });
-        // // this.peer.on("close", () => {
-        // // });
     }
 
     startSendingVideo(stream: MediaStream) {
@@ -291,8 +535,18 @@ class NetworkedVideo {
         };
     }
 
-    stopSendingVideo(_steam: MediaStream) {
-
+    stopSendingVideo(_steam: MediaStream | undefined | null) {
+        if (_steam) {
+            const calls = this._sendingVideoStreams.get(_steam);
+            if (calls) {
+                for (const call of calls) {
+                    call.close();
+                }
+            }
+            this._sendingVideoStreams.delete(_steam);
+            if (calls)
+                console.log("Currently sending", this._sendingVideoStreams);
+        }
     }
 
     private onConnectRoomFn: Function = this.onConnectRoom.bind(this);
@@ -300,14 +554,22 @@ class NetworkedVideo {
     private onUserLeftFn: Function = this.onUserLeft.bind(this);
 
     enable() {
-        this.context.connection.beginListen(PeerHandleEvent.Connected, this.onUserConnectedFn);
+        this.peer.enable();
+        this.context.connection.beginListen(PeerEvent.Connected, this.onUserConnectedFn);
+        this.peer.addEventListener("receive-video", this.onReceiveVideo.bind(this));
     }
 
     disable() {
+        this.peer.disable();
         // this.context.connection.stopListening(RoomEvents.UserJoinedRoom, this.onUserConnectedFn);
         // this.context.connection.stopListening(RoomEvents.UserLeftRoom, this.onUserLeftFn);
     }
 
+    private onReceiveVideo(evt) {
+        console.log("RECEIVE VIDEO", evt);
+        this.dispatchEvent({ type: "receive-video", target: this, stream: evt.stream, userId: evt.userId });
+    }
+
     private onConnectRoom() {
 
     }
@@ -316,41 +578,51 @@ class NetworkedVideo {
         // console.log(this.peer.id, user.guid)
         if (this.peer.id === user.guid) {
             console.log("USER CONNECTED", user);
-            const userId = this.peer.getUserIdFromPeerId(user.peerId);
-            console.log(userId);
-            // this.updateSendingCalls();
-            const call = this.peer.makeCall(user.peerId, this._sendingVideoStreams.keys().next().value);
-           
-            // for (const userId of this.context.connection.usersInRoom()) {
-            //     const id = this.peer.getPeerIdFromUserId(userId);
-            //     if(id === user.peerId) console.log(userId);
-            // }
+            const stream = this._sendingVideoStreams.keys().next().value;
+            this.peer.makeCall(user.peerId, stream);
         }
     }
 
     private onUserLeft(_: UserJoinedOrLeftRoomModel) {
-
+        this.stopCallsToUsersThatAreNotInTheRoomAnymore();
     }
 
     private updateSendingCalls() {
-        // for (const str of this._sendingVideoStreams.keys()) {
-        //     const calls = this._sendingVideoStreams.get(str) || [];
-        //     for (const userId of this.context.connection.usersInRoom()) {
-        //         const existing = calls.find(c => c.userId === userId);
-        //         if (!existing) {
-        //             const call = this.peer.makeCall(userId, str);
-        //             console.log(call);
-        //             if (call) {
-        //                 calls.push({
-        //                     call: call,
-        //                     stream: str,
-        //                     userId: userId
-        //                 });
-        //             }
-        //         }
-        //     }
-        //     this._sendingVideoStreams.set(str, calls);
-        // }
+        let startedNewCall = false;
+        for (const stream of this._sendingVideoStreams.keys()) {
+            const calls = this._sendingVideoStreams.get(stream) || [];
+            for (const userId of this.context.connection.usersInRoom()) {
+                if (userId === this.context.connection.connectionId) continue;
+                const existing = calls.find(c => c.userId === userId);
+                if (!existing) {
+                    const handle = this.peer.makeCall(this.peer.getPeerIdFromUserId(userId), stream);
+                    if (handle) {
+                        startedNewCall = true;
+                        calls.push(handle);
+                    }
+                }
+            }
+
+            this._sendingVideoStreams.set(stream, calls);
+        }
+        this.stopCallsToUsersThatAreNotInTheRoomAnymore();
+        if (startedNewCall) {
+            console.log("Currently sending", this._sendingVideoStreams);
+        }
+    }
+
+    private stopCallsToUsersThatAreNotInTheRoomAnymore() {
+        for (const stream of this._sendingVideoStreams.keys()) {
+            const calls = this._sendingVideoStreams.get(stream);
+            if (!calls) continue;
+            for (let i = calls.length - 1; i >= 0; i--) {
+                const call = calls[i];
+                if (!this.context.connection.userIsInRoom(call.userId)) {
+                    call.close();
+                    calls.splice(i, 1);
+                }
+            }
+        }
     }
 
     // const call = peer.call(peerId, stream);