@needle-tools/engine 2.29.0-pre → 2.30.0-pre

package/src/engine/extensions/extension_utils.ts

@@ -10,13 +10,20 @@ declare type DependencyInfo = {
     dependencyName: string,
 }
 
-const rootExtensionPrefix = "/extensions/";
+const rootExtensionPrefix = ["/extensions/", "extensions/"];
 const defaultDependencies = [
     { prefix: "/nodes/", dependencyName: "node" },
     { prefix: "/meshes/", dependencyName: "mesh" },
     { prefix: "/materials/", dependencyName: "material" },
     { prefix: "/textures/", dependencyName: "texture" },
-    { prefix: "/animations/", dependencyName: "animation" }
+    { prefix: "/animations/", dependencyName: "animation" },
+
+    // legacy support
+    { prefix: "nodes/", dependencyName: "node" },
+    { prefix: "meshes/", dependencyName: "mesh" },
+    { prefix: "materials/", dependencyName: "material" },
+    { prefix: "textures/", dependencyName: "texture" },
+    { prefix: "animations/", dependencyName: "animation" },
 ]
 
 export async function resolveReferences(parser: GLTFParser, obj) {
@@ -95,16 +102,21 @@ function internalResolve(paths: DependencyInfo[], parser: GLTFParser, obj, promi
 
 
 function resolveExtension(parser: GLTFParser, str): Promise<void> | null {
-    if (parser && parser.plugins && typeof str === "string" && str.startsWith(rootExtensionPrefix)) {
-        let name = str.substring(rootExtensionPrefix.length);
-        const endIndex = name.indexOf("/");
-        if (endIndex >= 0) name = name.substring(0, endIndex);
-        const ext = parser.plugins[name] as IExtensionReferenceResolver;
-        if (debugExtension)
-            console.log(name, ext);
-        if (typeof ext?.resolve === "function") {
-            const path = str.substring(rootExtensionPrefix.length + name.length + 1);
-            return ext.resolve(parser, path);
+    if (parser && parser.plugins && typeof str === "string") {
+        for (const prefix of rootExtensionPrefix) {
+            if (str.startsWith(prefix)) {
+                let name = str.substring(prefix.length);
+                const endIndex = name.indexOf("/");
+                if (endIndex >= 0) name = name.substring(0, endIndex);
+                const ext = parser.plugins[name] as IExtensionReferenceResolver;
+                if (debugExtension)
+                    console.log(name, ext);
+                if (typeof ext?.resolve === "function") {
+                    const path = str.substring(prefix.length + name.length + 1);
+                    return ext.resolve(parser, path);
+                }
+                break;
+            }
         }
     }
     return null;

package/src/engine/extensions/extensions.ts

@@ -26,8 +26,9 @@ export function registerComponentExtension(loader: GLTFLoader): NEEDLE_component
 class PointerResolver {
     resolvePath(path: string) {
         if (path.includes('/extensions/builtin_components/'))
-            path = path.replace('/extensions/builtin_components/', '/userData/components/');
-
+            return path.replace('/extensions/builtin_components/', '/userData/components/');
+        if (path.includes('extensions/builtin_components/'))
+            return path.replace('extensions/builtin_components/', '/userData/components/');
         return path;
     }
 }

package/src/engine-components/Camera.ts

@@ -5,6 +5,7 @@ import { getParam } from "../engine/engine_utils";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { RGBAColor } from "./js-extensions/RGBAColor";
 import { PerspectiveCamera } from "three";
+import { XRSessionMode } from "../engine/engine_setup";
 
 export enum ClearFlags {
     Skybox = 1,
@@ -192,13 +193,20 @@ export class Camera extends Behaviour {
     }
 
     private environmentIsTransparent(): boolean {
-        const session = this.context.renderer.xr?.getSession()
+        const session = this.context.renderer.xr?.getSession();
         if (!session) return false;
         const environmentBlendMode = session.environmentBlendMode;
         const transparent = environmentBlendMode === 'additive' || environmentBlendMode === 'alpha-blend';
+        // workaround for Quest 2 returning opaque when it should be alpha-blend
+        // check user agent if this is the Quest browser and return true if so
+
+        if (environmentBlendMode === "opaque" && navigator.userAgent?.includes("OculusBrowser")) {
+            if (this.context.xrSessionMode === XRSessionMode.ImmersiveAR) return true;
+        }
         return transparent;
     }
 
+
     private enableSkybox() {
         if (!this._skybox)
             this._skybox = new CameraSkybox(this);

package/src/engine-components/OrbitControls.ts

@@ -102,7 +102,8 @@ export class OrbitControls extends Behaviour {
         if (this._controls) {
             const camGo = GameObject.getComponent(this.gameObject, Camera);
             if (camGo && !this.setFromTargetPosition()) {
-                console.log("NO TARGET");
+                if(this.debugLog)
+                    console.log("NO TARGET");
                 const forward = new THREE.Vector3(0, 0, -1).applyMatrix4(camGo.cam.matrixWorld);
                 this.setTarget(forward, true);
             }

package/src/engine-components/Renderer.ts

@@ -13,6 +13,7 @@ const suppressInstancing = getParam("noInstancing");
 const debugLightmap = getParam("debuglightmaps") ? true : false;
 const debugInstancing = getParam("debuginstancing");
 const debugProgressiveLoading = getParam("debugprogressiveload");
+const suppressProgressiveLoading = getParam("noprogressiveload");
 
 export class FieldWithDefault {
     public path: string | null = null;
@@ -172,6 +173,8 @@ export class Renderer extends Behaviour {
     }
 
     awake() {
+        this.clearInstancingState();
+
         const type = this.gameObject.type;
         if (type === "Group") {
             for (const child of this.gameObject.children) {
@@ -245,6 +248,11 @@ export class Renderer extends Behaviour {
     private handles: InstanceHandle[] | null | undefined = undefined;
     private prevLayers: number[] | null | undefined = undefined;
 
+    private clearInstancingState() {
+        this._isInstancingEnabled = false;
+        this.handles = undefined;
+        this.prevLayers = undefined;
+    }
     setInstancingEnabled(enabled: boolean): boolean {
         if (this._isInstancingEnabled === enabled) return enabled && (this.handles === undefined || this.handles != null && this.handles.length > 0);
         this._isInstancingEnabled = enabled;
@@ -338,16 +346,16 @@ export class Renderer extends Behaviour {
     onBeforeRenderThree(_renderer, _scene, _camera, _geometry, material, _group) {
 
         // progressive load before rendering so we only load textures for visible materials
-        if (material._didRequestTextureLOD === undefined) {
+        if (!suppressProgressiveLoading && material._didRequestTextureLOD === undefined) {
             material._didRequestTextureLOD = 0;
             if (debugProgressiveLoading) {
                 console.log("Load material LOD (with delay)", material.name);
                 setTimeout(() => {
-                    NEEDLE_deferred_texture.assignTextureLOD(this.context, material);
+                    NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
                 }, 2000);
             }
             else {
-                NEEDLE_deferred_texture.assignTextureLOD(this.context, material);
+                NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
             }
         }
 

package/src/engine-components/ScreenCapture.ts

@@ -1,11 +1,19 @@
 import { Behaviour, GameObject } from "./Component";
 import { VideoPlayer } from "./VideoPlayer";
+import Peer from "peerjs"
+import { Context } from "../engine/engine_setup";
+import { RoomEvents } from "../engine/engine_networking";
+import { UserJoinedOrLeftRoomModel } from "../engine/engine_networking";
+import { serializeable } from "../engine/engine_serialization";
+import { IModel } from "../engine/engine_networking";
 
 export class ScreenCapture extends Behaviour {
 
+    @serializeable()
     streamOnAwake: boolean = true;
 
     start() {
+        console.warn("EXPERIMENTAL screen capture", this);
         if (this.streamOnAwake)
             this.open();
     }
@@ -22,7 +30,7 @@ export class ScreenCapture extends Behaviour {
             if (player) {
                 const displayMediaOptions: DisplayMediaStreamConstraints = {
                     video: true,
-                    audio: false
+                    audio: true
                 };
                 this.captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
                 if (this.captureStream && this.requestOpen) {
@@ -30,6 +38,7 @@ export class ScreenCapture extends Behaviour {
                     player.clip = this.captureStream;
                     player.create(true);
                 }
+                this.handleNetworkedVideo();
             }
         } catch (err) {
             console.error("Error: " + err);
@@ -44,4 +53,305 @@ export class ScreenCapture extends Behaviour {
             this.captureStream = null;
         }
     }
-}
+
+    private handleNetworkedVideo() {
+        if (!this.context.connection.isConnected) {
+            console.warn("Will not stream video - not connected to room");
+            return;
+        }
+        if (this.captureStream) {
+            const handle = PeerHandle.getOrCreate(this.context, this.guid);
+            handle.enable();
+            // if (!this.peer)
+            //     this.peer = new Peer();
+            const net = new NetworkedVideo(this.context, handle);
+            net.enable();
+            net.startSendingVideo(this.captureStream);
+        }
+    }
+
+}
+
+
+enum PeerHandleEvent {
+    Connected = "peer-user-connected",
+}
+
+class PeerUserConnectedModel implements IModel {
+    /** the peer handle id */
+    readonly guid: string;
+    readonly peerId: string;
+    // internal so server doesnt save it to persistent storage
+    readonly dontSave: boolean = true;
+    constructor(handle: PeerHandle, peerId: string) {
+        this.guid = handle.id;
+        this.peerId = peerId;
+    }
+}
+
+enum CallDirection {
+    Incoming = "incoming",
+    Outgoing = "outgoing",
+}
+
+class CallHandle {
+    readonly direction: CallDirection;
+    readonly call: Peer.MediaConnection;
+    stream: MediaStream | null;
+
+    constructor(call: Peer.MediaConnection, direction: CallDirection) {
+        this.call = call;
+        this.direction = direction;
+        this.stream = null;
+        call.on("stream", stream => {
+            this.stream = stream;
+        });
+    }
+}
+
+class PeerHandle {
+
+    private static readonly instances: Map<string, PeerHandle> = new Map();
+
+    static getOrCreate(context: Context, guid: string): PeerHandle {
+        // if (id === undefined) {
+        //     // randomId
+        //     id = Math.random().toFixed(5);
+        // }
+        if (PeerHandle.instances.has(guid))
+            return PeerHandle.instances.get(guid)!;
+        const peer = new PeerHandle(context, guid);
+        PeerHandle.instances.set(guid, peer);
+        return peer;
+    }
+
+    getPeerIdFromUserId(userConnectionId: string): string {
+        // we build the peer id ourselves so we dont need to wait for peer to report it
+        return this.id + "-" + userConnectionId;
+    }
+
+    getUserIdFromPeerId(peerId: string): string {
+        return peerId.substring(this.id.length + 1);
+    }
+
+    makeCall(peerId: string, stream: MediaStream): CallHandle | undefined {
+        // if (userId === this.context.connection.connectionId) {
+        //     console.warn("Can not call self");
+        //     return;
+        // }
+        // const peerId = this.getUserPeerId(userId);
+        console.log("CALL", peerId);
+        const call = this._peer?.call(peerId, stream);
+        if (call)
+            return this.registerCall(call, CallDirection.Outgoing);
+        return undefined;
+    }
+
+    get peer(): Peer | undefined { return this._peer; }
+
+    readonly id: string;
+    readonly context: Context;
+    private _peer: Peer | undefined;
+    private _incomingCalls: CallHandle[] = [];
+    private _outgoingCalls: CallHandle[] = [];
+
+    private constructor(context: Context, id: string) {
+        this.context = context;
+        this.id = id;
+        this.setupPeer();
+        navigator["getUserMedia"] = (
+            navigator["getUserMedia"] || navigator["webkitGetUserMedia"] ||
+            navigator["mozGetUserMedia"] || navigator["msGetUserMedia"]
+        );
+    }
+
+    private _enabled: boolean = false;
+    private _enabledPeer: boolean = false;
+    private onConnectRoomFn: Function = this.onConnectRoom.bind(this);
+    private onUserJoinedOrLeftRoomFn: Function = this.onUserJoinedOrLeftRoom.bind(this);
+    private onPeerConnectFn: (id) => void = this.onPeerConnect.bind(this);
+    private onPeerReceiveCallFn: (call) => void = this.onPeerReceivingCall.bind(this);
+    // private _connectionPeerIdMap : Map<string, string> = new Map();
+
+    enable() {
+        if (this._enabled) return;
+        this._enabled = true;
+        this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onConnectRoomFn);
+        this.context.connection.beginListen(RoomEvents.UserJoinedRoom, this.onUserJoinedOrLeftRoomFn);
+        this.context.connection.beginListen(RoomEvents.UserLeftRoom, this.onUserJoinedOrLeftRoomFn);
+        this.subscribePeerEvents();
+    }
+
+    disable() {
+        if (!this._enabled) return;
+        this._enabled = false;
+        this.context.connection.stopListening(RoomEvents.JoinedRoom, this.onConnectRoomFn);
+        this.context.connection.stopListening(RoomEvents.UserJoinedRoom, this.onUserJoinedOrLeftRoomFn);
+        this.context.connection.stopListening(RoomEvents.UserLeftRoom, this.onUserJoinedOrLeftRoomFn);
+        this.unsubscribePeerEvents();
+    }
+
+    private onConnectRoom(): void {
+        this.setupPeer();
+    };
+
+    private onUserJoinedOrLeftRoom(_: UserJoinedOrLeftRoomModel): void {
+    };
+
+    private setupPeer() {
+        if (!this.context.connection.connectionId) return;
+        if (this._enabledPeer) return;
+        this._enabledPeer = true;
+        if (!this._peer) {
+            const peerId = this.getPeerIdFromUserId(this.context.connection.connectionId);
+            this._peer = new Peer(peerId);
+        }
+        if (this._enabled)
+            this.subscribePeerEvents();
+    }
+
+    private subscribePeerEvents() {
+        if (!this._peer) return;
+        this._peer.on("open", this.onPeerConnectFn);
+        this._peer.on("call", this.onPeerReceiveCallFn);
+        // this.context.connection.beginListen(PeerEvent.Connected, this.onRemotePeerConnect.bind(this));
+    }
+
+    private unsubscribePeerEvents() {
+    }
+
+    private onPeerConnect(id): void {
+        console.log("connect", id);
+        this.context.connection.send(PeerHandleEvent.Connected, new PeerUserConnectedModel(this, id));
+    }
+
+    private onPeerReceivingCall(call: Peer.MediaConnection): void {
+        console.log("RECEIVE CALL");
+        call.answer();
+        this.registerCall(call, CallDirection.Incoming);
+    }
+
+    private registerCall(call: Peer.MediaConnection, direction: CallDirection): CallHandle {
+        const arr = direction === CallDirection.Incoming ? this._incomingCalls : this._outgoingCalls;
+        const handle = new CallHandle(call, direction);
+        arr.push(handle);
+        call.on("error", err => {
+            console.error(err);
+        });
+        call.on("close", () => {
+            const index = arr.indexOf(handle);
+            if (index !== -1)
+                arr.splice(index, 1);
+        });
+        return handle;
+    }
+
+    // private onRemotePeerConnect(user: PeerUserConnectedModel) {
+    //     console.log("other user connected", user);
+    // }
+}
+
+
+type UserVideoCall = {
+    call: Peer.MediaConnection;
+    stream: MediaStream;
+    userId: string;
+}
+
+class NetworkedVideo {
+
+    private readonly context: Context;
+    private readonly peer: PeerHandle;
+
+    private _sendingVideoStreams: Map<MediaStream, UserVideoCall[]> = new Map();
+
+    constructor(context: Context, peer: PeerHandle) {
+        this.context = context;
+        this.peer = peer;
+
+        // this.peer.on("open", (id) => {
+        //     console.log("PEERID", id);
+        // });
+        // // this.peer.on("call", (call) => {
+        // // });
+        // // this.peer.on("connection", (conn) => {
+        // // });
+        // // this.peer.on("error", (err) => {
+        // // });
+        // // this.peer.on("disconnected", () => {
+        // // });
+        // // this.peer.on("close", () => {
+        // // });
+    }
+
+    startSendingVideo(stream: MediaStream) {
+        if (!this._sendingVideoStreams.has(stream)) {
+            this._sendingVideoStreams.set(stream, []);
+            this.updateSendingCalls();
+        };
+    }
+
+    stopSendingVideo(_steam: MediaStream) {
+
+    }
+
+    private onConnectRoomFn: Function = this.onConnectRoom.bind(this);
+    private onUserConnectedFn: Function = this.onUserConnected.bind(this);
+    private onUserLeftFn: Function = this.onUserLeft.bind(this);
+
+    enable() {
+        this.context.connection.beginListen(PeerHandleEvent.Connected, this.onUserConnectedFn);
+    }
+
+    disable() {
+        // this.context.connection.stopListening(RoomEvents.UserJoinedRoom, this.onUserConnectedFn);
+        // this.context.connection.stopListening(RoomEvents.UserLeftRoom, this.onUserLeftFn);
+    }
+
+    private onConnectRoom() {
+
+    }
+
+    private onUserConnected(user: PeerUserConnectedModel) {
+        // console.log(this.peer.id, user.guid)
+        if (this.peer.id === user.guid) {
+            console.log("USER CONNECTED", user);
+            const userId = this.peer.getUserIdFromPeerId(user.peerId);
+            console.log(userId);
+            // this.updateSendingCalls();
+            const call = this.peer.makeCall(user.peerId, this._sendingVideoStreams.keys().next().value);
+           
+            // for (const userId of this.context.connection.usersInRoom()) {
+            //     const id = this.peer.getPeerIdFromUserId(userId);
+            //     if(id === user.peerId) console.log(userId);
+            // }
+        }
+    }
+
+    private onUserLeft(_: UserJoinedOrLeftRoomModel) {
+
+    }
+
+    private updateSendingCalls() {
+        // for (const str of this._sendingVideoStreams.keys()) {
+        //     const calls = this._sendingVideoStreams.get(str) || [];
+        //     for (const userId of this.context.connection.usersInRoom()) {
+        //         const existing = calls.find(c => c.userId === userId);
+        //         if (!existing) {
+        //             const call = this.peer.makeCall(userId, str);
+        //             console.log(call);
+        //             if (call) {
+        //                 calls.push({
+        //                     call: call,
+        //                     stream: str,
+        //                     userId: userId
+        //                 });
+        //             }
+        //         }
+        //     }
+        //     this._sendingVideoStreams.set(str, calls);
+        // }
+    }
+
+    // const call = peer.call(peerId, stream);
+}

package/src/engine-components/SpectatorCamera.ts

@@ -170,6 +170,7 @@ export class SpectatorCamera extends Behaviour {
 
     private onXRSessionStart(_evt) {
         if (!this.isSupportedPlatform()) return;
+        if (debug) console.log(this.context.mainCamera);
         if (this.context.mainCamera) {
             this.followSelf();
         }
@@ -188,6 +189,7 @@ export class SpectatorCamera extends Behaviour {
 
     private followSelf() {
         this.target = this.context.players.getPlayerView(this.context.connection.connectionId);
+        if (debug) console.log("Follow self", this.target);
     }
 
     // TODO: only show Spectator cam for DesktopVR;
@@ -219,6 +221,8 @@ export class SpectatorCamera extends Behaviour {
 
         this.setAvatarFlagsBeforeRender();
 
+        const mainCam = this.context.mainCameraComponent;
+
         // these should not be needed if we don't override viewport/scissor
         // renderer.getViewport(this.currentViewport);
         // renderer.getScissor(this.currentScissor);
@@ -231,7 +235,17 @@ export class SpectatorCamera extends Behaviour {
         // renderer.setViewport(left, bottom, width, height);
         // renderer.setScissor(left, bottom, width, height);
         // renderer.setScissorTest(true);
-        renderer.setClearColor(new THREE.Color(1, 1, 1));
+        if (mainCam) {
+            const backgroundColor = mainCam.backgroundColor;
+            if (backgroundColor)
+                renderer.setClearColor(backgroundColor, backgroundColor.alpha);
+            this.cam.backgroundColor = backgroundColor;
+            this.cam.clearFlags = mainCam.clearFlags;
+            this.cam.nearClipPlane = mainCam.nearClipPlane;
+            this.cam.farClipPlane = mainCam.farClipPlane;
+        }
+        else
+            renderer.setClearColor(new THREE.Color(1, 1, 1));
         renderer.setRenderTarget(null); // null: direct to Canvas
         renderer.xr.enabled = false;
         const cam = this.cam?.cam;
@@ -384,8 +398,16 @@ class SpectatorHandler implements ISpectatorHandler {
         if (!target || !this.follow) return;
         switch (mode) {
             case SpectatorMode.FirstPerson:
-                this.follow.followFactor = 20;
-                this.follow.rotateFactor = 20;
+                if (this.view?.viewDevice !== ViewDevice.Browser) {
+                    // soft follow for AR and VR
+                    this.follow.followFactor = 5;
+                    this.follow.rotateFactor = 5;
+                }
+                else {
+                    // snappy follow for desktop
+                    this.follow.followFactor = 50;
+                    this.follow.rotateFactor = 50;
+                }
                 target.position.set(0, 0, 0);
                 break;
             case SpectatorMode.ThirdPerson:
@@ -427,10 +449,11 @@ class SpectatorSelectionController {
             downTime = this.context.time.time;
         });
         this.context.input.addEventListener(InputEvents.PointerUp, _ => {
-            if (this.context.time.time - downTime > 1) {
+            const dt = this.context.time.time - downTime;
+            if (dt > 1) {
                 this.spectator.stopSpectating();
             }
-            else
+            else if (this.context.input.getPointerClicked(0) && dt < .3)
                 this.trySelectObject();
         });
     }

package/src/engine-components/SyncedRoom.ts

@@ -44,6 +44,7 @@ export class SyncedRoom extends Behaviour {
     }
 
     tryJoinRoom(call: number = 0): boolean {
+        if(call === undefined) call = 0;
         let hasRoomParameter = false;
         if (this.urlParameterName) {
             const val = utils.getParam(this.urlParameterName);