@needle-tools/engine 2.28.0-pre → 2.30.0-pre

package/src/engine-components/Camera.ts

@@ -5,6 +5,7 @@ import { getParam } from "../engine/engine_utils";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { RGBAColor } from "./js-extensions/RGBAColor";
 import { PerspectiveCamera } from "three";
+import { XRSessionMode } from "../engine/engine_setup";
 
 export enum ClearFlags {
     Skybox = 1,
@@ -192,13 +193,20 @@ export class Camera extends Behaviour {
     }
 
     private environmentIsTransparent(): boolean {
-        const session = this.context.renderer.xr?.getSession()
+        const session = this.context.renderer.xr?.getSession();
         if (!session) return false;
         const environmentBlendMode = session.environmentBlendMode;
         const transparent = environmentBlendMode === 'additive' || environmentBlendMode === 'alpha-blend';
+        // workaround for Quest 2 returning opaque when it should be alpha-blend
+        // check user agent if this is the Quest browser and return true if so
+
+        if (environmentBlendMode === "opaque" && navigator.userAgent?.includes("OculusBrowser")) {
+            if (this.context.xrSessionMode === XRSessionMode.ImmersiveAR) return true;
+        }
         return transparent;
     }
 
+
     private enableSkybox() {
         if (!this._skybox)
             this._skybox = new CameraSkybox(this);

package/src/engine-components/Component.ts

@@ -644,7 +644,7 @@ abstract class GameObject extends THREE.Object3D implements THREE.Object3D {
         return object.getComponentsInChildren(typeName, go, arr);
     }
 
-    public static getComponentInParent<T>(go: GameObject | THREE.Object3D, typeName: Constructor<T>): T | null {
+    public static getComponentInParent<T>(go: GameObject | THREE.Object3D | Array<Object3D>, typeName: Constructor<T>): T | null {
         return object.getComponentInParent(typeName, go);
     }
 

package/src/engine-components/Light.ts

@@ -334,7 +334,10 @@ export class Light extends Behaviour {
             }
         }
 
+
         if (this.light !== undefined) {
+            this._intensity = this.light.intensity;
+            
             if (this.shadows !== LightShadows.None) {
                 this.light.castShadow = true;
             }

package/src/engine-components/OrbitControls.ts

@@ -9,7 +9,6 @@ import { serializeable } from "../engine/engine_serialization_decorator";
 import { Vector3 } from "three";
 
 export class OrbitControls extends Behaviour {
-
     public get controls() {
         return this._controls;
     }
@@ -31,7 +30,7 @@ export class OrbitControls extends Behaviour {
     minZoom: number = 0;
     maxZoom: number = Infinity;
     enablePan: boolean = true;
-    @serializeable(LookAtConstraint)
+    @serializeable()
     lookAtConstraint: LookAtConstraint | null = null;
     lookAtConstraint01: number = 1;
     middleClickToFocus: boolean = true;
@@ -103,7 +102,8 @@ export class OrbitControls extends Behaviour {
         if (this._controls) {
             const camGo = GameObject.getComponent(this.gameObject, Camera);
             if (camGo && !this.setFromTargetPosition()) {
-                console.log("NO TARGET");
+                if(this.debugLog)
+                    console.log("NO TARGET");
                 const forward = new THREE.Vector3(0, 0, -1).applyMatrix4(camGo.cam.matrixWorld);
                 this.setTarget(forward, true);
             }

package/src/engine-components/ParticleSystem.ts

@@ -61,7 +61,7 @@ export class ParticleSystem extends Behaviour {
 
     awake(): void {
         this.renderer = GameObject.getComponent(this.gameObject, ParticleSystemRenderer)!;
-        if (debug)
+        // if (debug)
             console.log(this);
     }
 
@@ -108,31 +108,30 @@ export class ParticleSystem extends Behaviour {
         this.activeCount = this.main.prewarm ? this.main.maxParticles : 0;
     }
 
-    update() {
-        if (!this.geometry) return;
-
-        const t = this.context.time.deltaTime;
-        this.emit(t);
-
-        for (let i = 0; i < this.activeCount; i += 1) {
-            const ps = this.particleStates[i];
-            if (!ps) continue;
-            const vx = ps.velocity.x * t;
-            const vy = ps.velocity.y * t;
-            const vz = ps.velocity.z * t;
-            ps.position.x += vx;
-            ps.position.y += vy;
-            ps.position.z += vz;
-            this.updateOverLifetime(i, ps);
-            this.geometry.attributes.position.setXYZ(i, ps.position.x, ps.position.y, ps.position.z);
-            this.geometry.attributes.color.setXYZ(i, ps.color.r, ps.color.g, ps.color.b);
-        }
-        if (this.geometry) {
-            this.geometry.attributes.position.needsUpdate = true;
-            this.geometry.setDrawRange(0, this.activeCount);
-        }
-        // console.log(this.activeCount);
-    }
+    // update() {
+    //     if (!this.geometry) return;
+
+    //     const t = this.context.time.deltaTime;
+    //     this.emit(t);
+
+    //     for (let i = 0; i < this.activeCount; i += 1) {
+    //         const ps = this.particleStates[i];
+    //         if (!ps) continue;
+    //         const vx = ps.velocity.x * t;
+    //         const vy = ps.velocity.y * t;
+    //         const vz = ps.velocity.z * t;
+    //         ps.position.x += vx;
+    //         ps.position.y += vy;
+    //         ps.position.z += vz;
+    //         this.updateOverLifetime(i, ps);
+    //         this.geometry.attributes.position.setXYZ(i, ps.position.x, ps.position.y, ps.position.z);
+    //         this.geometry.attributes.color.setXYZ(i, ps.color.r, ps.color.g, ps.color.b);
+    //     }
+    //     if (this.geometry) {
+    //         this.geometry.attributes.position.needsUpdate = true;
+    //         this.geometry.setDrawRange(0, this.activeCount);
+    //     }
+    // }
 
     emit(deltaTime: number) {
         const count = this.activeCount;

package/src/engine-components/PlayerColor.ts

@@ -68,7 +68,7 @@ export class PlayerColor extends Behaviour {
             mesh.material = mat;
             // console.log("CLONE", mat);
         }
-        else console.log("DONT CLONE", mat);
+        // else console.log("DONT CLONE", mat);
         mat["color"] = col;
     }
 

package/src/engine-components/Renderer.ts

@@ -13,6 +13,7 @@ const suppressInstancing = getParam("noInstancing");
 const debugLightmap = getParam("debuglightmaps") ? true : false;
 const debugInstancing = getParam("debuginstancing");
 const debugProgressiveLoading = getParam("debugprogressiveload");
+const suppressProgressiveLoading = getParam("noprogressiveload");
 
 export class FieldWithDefault {
     public path: string | null = null;
@@ -172,6 +173,8 @@ export class Renderer extends Behaviour {
     }
 
     awake() {
+        this.clearInstancingState();
+
         const type = this.gameObject.type;
         if (type === "Group") {
             for (const child of this.gameObject.children) {
@@ -245,6 +248,11 @@ export class Renderer extends Behaviour {
     private handles: InstanceHandle[] | null | undefined = undefined;
     private prevLayers: number[] | null | undefined = undefined;
 
+    private clearInstancingState() {
+        this._isInstancingEnabled = false;
+        this.handles = undefined;
+        this.prevLayers = undefined;
+    }
     setInstancingEnabled(enabled: boolean): boolean {
         if (this._isInstancingEnabled === enabled) return enabled && (this.handles === undefined || this.handles != null && this.handles.length > 0);
         this._isInstancingEnabled = enabled;
@@ -338,16 +346,16 @@ export class Renderer extends Behaviour {
     onBeforeRenderThree(_renderer, _scene, _camera, _geometry, material, _group) {
 
         // progressive load before rendering so we only load textures for visible materials
-        if (material._didRequestTextureLOD === undefined) {
+        if (!suppressProgressiveLoading && material._didRequestTextureLOD === undefined) {
             material._didRequestTextureLOD = 0;
             if (debugProgressiveLoading) {
                 console.log("Load material LOD (with delay)", material.name);
                 setTimeout(() => {
-                    NEEDLE_deferred_texture.assignTextureLOD(this.context, material);
+                    NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
                 }, 2000);
             }
             else {
-                NEEDLE_deferred_texture.assignTextureLOD(this.context, material);
+                NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
             }
         }
 

package/src/engine-components/ScreenCapture.ts

@@ -1,11 +1,19 @@
 import { Behaviour, GameObject } from "./Component";
 import { VideoPlayer } from "./VideoPlayer";
+import Peer from "peerjs"
+import { Context } from "../engine/engine_setup";
+import { RoomEvents } from "../engine/engine_networking";
+import { UserJoinedOrLeftRoomModel } from "../engine/engine_networking";
+import { serializeable } from "../engine/engine_serialization";
+import { IModel } from "../engine/engine_networking";
 
 export class ScreenCapture extends Behaviour {
 
+    @serializeable()
     streamOnAwake: boolean = true;
 
     start() {
+        console.warn("EXPERIMENTAL screen capture", this);
         if (this.streamOnAwake)
             this.open();
     }
@@ -22,7 +30,7 @@ export class ScreenCapture extends Behaviour {
             if (player) {
                 const displayMediaOptions: DisplayMediaStreamConstraints = {
                     video: true,
-                    audio: false
+                    audio: true
                 };
                 this.captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
                 if (this.captureStream && this.requestOpen) {
@@ -30,6 +38,7 @@ export class ScreenCapture extends Behaviour {
                     player.clip = this.captureStream;
                     player.create(true);
                 }
+                this.handleNetworkedVideo();
             }
         } catch (err) {
             console.error("Error: " + err);
@@ -44,4 +53,305 @@ export class ScreenCapture extends Behaviour {
             this.captureStream = null;
         }
     }
-}
+
+    private handleNetworkedVideo() {
+        if (!this.context.connection.isConnected) {
+            console.warn("Will not stream video - not connected to room");
+            return;
+        }
+        if (this.captureStream) {
+            const handle = PeerHandle.getOrCreate(this.context, this.guid);
+            handle.enable();
+            // if (!this.peer)
+            //     this.peer = new Peer();
+            const net = new NetworkedVideo(this.context, handle);
+            net.enable();
+            net.startSendingVideo(this.captureStream);
+        }
+    }
+
+}
+
+
+enum PeerHandleEvent {
+    Connected = "peer-user-connected",
+}
+
+class PeerUserConnectedModel implements IModel {
+    /** the peer handle id */
+    readonly guid: string;
+    readonly peerId: string;
+    // internal so server doesnt save it to persistent storage
+    readonly dontSave: boolean = true;
+    constructor(handle: PeerHandle, peerId: string) {
+        this.guid = handle.id;
+        this.peerId = peerId;
+    }
+}
+
+enum CallDirection {
+    Incoming = "incoming",
+    Outgoing = "outgoing",
+}
+
+class CallHandle {
+    readonly direction: CallDirection;
+    readonly call: Peer.MediaConnection;
+    stream: MediaStream | null;
+
+    constructor(call: Peer.MediaConnection, direction: CallDirection) {
+        this.call = call;
+        this.direction = direction;
+        this.stream = null;
+        call.on("stream", stream => {
+            this.stream = stream;
+        });
+    }
+}
+
+class PeerHandle {
+
+    private static readonly instances: Map<string, PeerHandle> = new Map();
+
+    static getOrCreate(context: Context, guid: string): PeerHandle {
+        // if (id === undefined) {
+        //     // randomId
+        //     id = Math.random().toFixed(5);
+        // }
+        if (PeerHandle.instances.has(guid))
+            return PeerHandle.instances.get(guid)!;
+        const peer = new PeerHandle(context, guid);
+        PeerHandle.instances.set(guid, peer);
+        return peer;
+    }
+
+    getPeerIdFromUserId(userConnectionId: string): string {
+        // we build the peer id ourselves so we dont need to wait for peer to report it
+        return this.id + "-" + userConnectionId;
+    }
+
+    getUserIdFromPeerId(peerId: string): string {
+        return peerId.substring(this.id.length + 1);
+    }
+
+    makeCall(peerId: string, stream: MediaStream): CallHandle | undefined {
+        // if (userId === this.context.connection.connectionId) {
+        //     console.warn("Can not call self");
+        //     return;
+        // }
+        // const peerId = this.getUserPeerId(userId);
+        console.log("CALL", peerId);
+        const call = this._peer?.call(peerId, stream);
+        if (call)
+            return this.registerCall(call, CallDirection.Outgoing);
+        return undefined;
+    }
+
+    get peer(): Peer | undefined { return this._peer; }
+
+    readonly id: string;
+    readonly context: Context;
+    private _peer: Peer | undefined;
+    private _incomingCalls: CallHandle[] = [];
+    private _outgoingCalls: CallHandle[] = [];
+
+    private constructor(context: Context, id: string) {
+        this.context = context;
+        this.id = id;
+        this.setupPeer();
+        navigator["getUserMedia"] = (
+            navigator["getUserMedia"] || navigator["webkitGetUserMedia"] ||
+            navigator["mozGetUserMedia"] || navigator["msGetUserMedia"]
+        );
+    }
+
+    private _enabled: boolean = false;
+    private _enabledPeer: boolean = false;
+    private onConnectRoomFn: Function = this.onConnectRoom.bind(this);
+    private onUserJoinedOrLeftRoomFn: Function = this.onUserJoinedOrLeftRoom.bind(this);
+    private onPeerConnectFn: (id) => void = this.onPeerConnect.bind(this);
+    private onPeerReceiveCallFn: (call) => void = this.onPeerReceivingCall.bind(this);
+    // private _connectionPeerIdMap : Map<string, string> = new Map();
+
+    enable() {
+        if (this._enabled) return;
+        this._enabled = true;
+        this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onConnectRoomFn);
+        this.context.connection.beginListen(RoomEvents.UserJoinedRoom, this.onUserJoinedOrLeftRoomFn);
+        this.context.connection.beginListen(RoomEvents.UserLeftRoom, this.onUserJoinedOrLeftRoomFn);
+        this.subscribePeerEvents();
+    }
+
+    disable() {
+        if (!this._enabled) return;
+        this._enabled = false;
+        this.context.connection.stopListening(RoomEvents.JoinedRoom, this.onConnectRoomFn);
+        this.context.connection.stopListening(RoomEvents.UserJoinedRoom, this.onUserJoinedOrLeftRoomFn);
+        this.context.connection.stopListening(RoomEvents.UserLeftRoom, this.onUserJoinedOrLeftRoomFn);
+        this.unsubscribePeerEvents();
+    }
+
+    private onConnectRoom(): void {
+        this.setupPeer();
+    };
+
+    private onUserJoinedOrLeftRoom(_: UserJoinedOrLeftRoomModel): void {
+    };
+
+    private setupPeer() {
+        if (!this.context.connection.connectionId) return;
+        if (this._enabledPeer) return;
+        this._enabledPeer = true;
+        if (!this._peer) {
+            const peerId = this.getPeerIdFromUserId(this.context.connection.connectionId);
+            this._peer = new Peer(peerId);
+        }
+        if (this._enabled)
+            this.subscribePeerEvents();
+    }
+
+    private subscribePeerEvents() {
+        if (!this._peer) return;
+        this._peer.on("open", this.onPeerConnectFn);
+        this._peer.on("call", this.onPeerReceiveCallFn);
+        // this.context.connection.beginListen(PeerEvent.Connected, this.onRemotePeerConnect.bind(this));
+    }
+
+    private unsubscribePeerEvents() {
+    }
+
+    private onPeerConnect(id): void {
+        console.log("connect", id);
+        this.context.connection.send(PeerHandleEvent.Connected, new PeerUserConnectedModel(this, id));
+    }
+
+    private onPeerReceivingCall(call: Peer.MediaConnection): void {
+        console.log("RECEIVE CALL");
+        call.answer();
+        this.registerCall(call, CallDirection.Incoming);
+    }
+
+    private registerCall(call: Peer.MediaConnection, direction: CallDirection): CallHandle {
+        const arr = direction === CallDirection.Incoming ? this._incomingCalls : this._outgoingCalls;
+        const handle = new CallHandle(call, direction);
+        arr.push(handle);
+        call.on("error", err => {
+            console.error(err);
+        });
+        call.on("close", () => {
+            const index = arr.indexOf(handle);
+            if (index !== -1)
+                arr.splice(index, 1);
+        });
+        return handle;
+    }
+
+    // private onRemotePeerConnect(user: PeerUserConnectedModel) {
+    //     console.log("other user connected", user);
+    // }
+}
+
+
+type UserVideoCall = {
+    call: Peer.MediaConnection;
+    stream: MediaStream;
+    userId: string;
+}
+
+class NetworkedVideo {
+
+    private readonly context: Context;
+    private readonly peer: PeerHandle;
+
+    private _sendingVideoStreams: Map<MediaStream, UserVideoCall[]> = new Map();
+
+    constructor(context: Context, peer: PeerHandle) {
+        this.context = context;
+        this.peer = peer;
+
+        // this.peer.on("open", (id) => {
+        //     console.log("PEERID", id);
+        // });
+        // // this.peer.on("call", (call) => {
+        // // });
+        // // this.peer.on("connection", (conn) => {
+        // // });
+        // // this.peer.on("error", (err) => {
+        // // });
+        // // this.peer.on("disconnected", () => {
+        // // });
+        // // this.peer.on("close", () => {
+        // // });
+    }
+
+    startSendingVideo(stream: MediaStream) {
+        if (!this._sendingVideoStreams.has(stream)) {
+            this._sendingVideoStreams.set(stream, []);
+            this.updateSendingCalls();
+        };
+    }
+
+    stopSendingVideo(_steam: MediaStream) {
+
+    }
+
+    private onConnectRoomFn: Function = this.onConnectRoom.bind(this);
+    private onUserConnectedFn: Function = this.onUserConnected.bind(this);
+    private onUserLeftFn: Function = this.onUserLeft.bind(this);
+
+    enable() {
+        this.context.connection.beginListen(PeerHandleEvent.Connected, this.onUserConnectedFn);
+    }
+
+    disable() {
+        // this.context.connection.stopListening(RoomEvents.UserJoinedRoom, this.onUserConnectedFn);
+        // this.context.connection.stopListening(RoomEvents.UserLeftRoom, this.onUserLeftFn);
+    }
+
+    private onConnectRoom() {
+
+    }
+
+    private onUserConnected(user: PeerUserConnectedModel) {
+        // console.log(this.peer.id, user.guid)
+        if (this.peer.id === user.guid) {
+            console.log("USER CONNECTED", user);
+            const userId = this.peer.getUserIdFromPeerId(user.peerId);
+            console.log(userId);
+            // this.updateSendingCalls();
+            const call = this.peer.makeCall(user.peerId, this._sendingVideoStreams.keys().next().value);
+           
+            // for (const userId of this.context.connection.usersInRoom()) {
+            //     const id = this.peer.getPeerIdFromUserId(userId);
+            //     if(id === user.peerId) console.log(userId);
+            // }
+        }
+    }
+
+    private onUserLeft(_: UserJoinedOrLeftRoomModel) {
+
+    }
+
+    private updateSendingCalls() {
+        // for (const str of this._sendingVideoStreams.keys()) {
+        //     const calls = this._sendingVideoStreams.get(str) || [];
+        //     for (const userId of this.context.connection.usersInRoom()) {
+        //         const existing = calls.find(c => c.userId === userId);
+        //         if (!existing) {
+        //             const call = this.peer.makeCall(userId, str);
+        //             console.log(call);
+        //             if (call) {
+        //                 calls.push({
+        //                     call: call,
+        //                     stream: str,
+        //                     userId: userId
+        //                 });
+        //             }
+        //         }
+        //     }
+        //     this._sendingVideoStreams.set(str, calls);
+        // }
+    }
+
+    // const call = peer.call(peerId, stream);
+}