@needle-tools/engine 2.28.0-pre → 2.30.0-pre

package/src/engine-components/WebXR.ts

@@ -4,7 +4,7 @@ import { ARButton } from '../include/three/ARButton.js';
 import { VRButton } from '../include/three/VRButton.js';
 
 import * as THREE from 'three'
-import { ArrayCamera, XRHitTestSource } from 'three';
+import { ArrayCamera, Matrix4, Vector3, XRHitTestSource } from 'three';
 
 import * as utils from "../engine/engine_utils";
 import { noVoip } from "./Voip";
@@ -17,6 +17,7 @@ import { EngineElement } from "../engine/engine_element";
 import { AssetReference } from "../engine/engine_addressables";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { WebXRSync } from "./WebXRSync";
+import { XRSessionMode } from "../engine/engine_setup";
 
 
 export async function detectARSupport() {
@@ -73,7 +74,7 @@ export class WebXR extends Behaviour {
         this.events.addEventListener(type, listener);
         return listener;
     }
-    public static removeEventListener(type: string, listener: any) : any {
+    public static removeEventListener(type: string, listener: any): any {
         this.events.removeEventListener(type, listener);
         return listener;
     }
@@ -94,10 +95,10 @@ export class WebXR extends Behaviour {
 
     public static createARButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement {
         const domOverlayRoot = webXR.webAR?.getAROverlayContainer();
-        const features: any = { requiredFeatures: ['hit-test'] };
+        const features: any = {};
         if (domOverlayRoot) {
             features.domOverlay = { root: domOverlayRoot };
-            features.optionalFeatures = ['dom-overlay'];
+            features.optionalFeatures = ['hit-test', 'dom-overlay'];
         }
         const arButton = ARButton.createButton(webXR.context.renderer, features);
         arButton.classList.add('webxr-ar-button');
@@ -120,7 +121,8 @@ export class WebXR extends Behaviour {
         if (session) session.end();
     }
 
-    public get Rig(): THREE.Object3D | null {
+    public get Rig(): THREE.Object3D {
+        if(!this.rig) this.ensureRig();
         return this.rig;
     }
 
@@ -256,8 +258,6 @@ export class WebXR extends Behaviour {
         WebXR.events.dispatchEvent({ type: WebXREvent.XRUpdate, frame: frame, xr: this.context.renderer.xr, rig: this.rig });
     }
 
-
-
     private onClickedARButton() {
         if (!this._isInAR) {
             this._requestedAR = true;
@@ -313,6 +313,7 @@ export class WebXR extends Behaviour {
             }
             else {
                 this.rig = new THREE.Group();
+                this.rig.rotateY(Math.PI);
                 this.rig.name = "XRRig";
                 this.context.scene.add(this.rig);
             }
@@ -360,10 +361,12 @@ export class WebXR extends Behaviour {
 
         switch (flag) {
             case XRStateFlag.AR:
+                this.context.xrSessionMode = XRSessionMode.ImmersiveAR;
                 this._isInAR = true;
                 this.webAR?.onBegin(session);
                 break;
             case XRStateFlag.VR:
+                this.context.xrSessionMode = XRSessionMode.ImmersiveVR;
                 this._isInVR = true;
                 this.onEnterVR(session);
                 break;
@@ -390,6 +393,7 @@ export class WebXR extends Behaviour {
         this._isInVR = false;
         this._requestedAR = false;
         this._requestedVR = false;
+        this.context.xrSessionMode = undefined;
 
         if (this.xrMirrorWindow) {
             this.xrMirrorWindow.close();
@@ -447,7 +451,10 @@ export class WebXR extends Behaviour {
 // not sure if this should be a behaviour. 
 // for now we dont really need it to go through the usual update loop
 export class WebAR {
-    private webxr: WebXR;
+
+    get webxr(): WebXR { return this._webxr; }
+
+    private _webxr: WebXR;
 
     private reticle: THREE.Object3D | null = null;
     private hitTestSource: XRHitTestSource | null = null;
@@ -467,11 +474,13 @@ export class WebAR {
     private get context() { return this.webxr.context; }
 
     constructor(webxr: WebXR) {
-        this.webxr = webxr;
+        this._webxr = webxr;
     }
 
     private arDomOverlay: EngineElement | null = null;
     private arOverlayElement: HTMLElement | null = null;
+    private noHitTestAvailable: boolean = false;
+    private didPlaceARSessionRoot: boolean = false;
 
     getAROverlayContainer(): HTMLElement | null {
         this.arDomOverlay = this.webxr.context.domElement as EngineElement;
@@ -483,12 +492,14 @@ export class WebAR {
         this.reticleActive = active;
     }
 
-    onBegin(session: THREE.XRSession) {
+    async onBegin(session: THREE.XRSession) {
         const context = this.webxr.context;
         this.reticleActive = true;
+        this.didPlaceARSessionRoot = false;
 
+        const deviceType = navigator.userAgent?.includes("OculusBrowser") ? ControllerType.PhysicalDevice : ControllerType.Touch;
         for (let i = 0; i < 4; i++) {
-            WebXRController.Create(this.webxr, i, this.webxr.gameObject, ControllerType.Touch)
+            WebXRController.Create(this.webxr, i, this.webxr.gameObject, deviceType)
         }
 
         if (!this.sessionRoot || this.sessionRoot.destroyed || !this.sessionRoot.activeAndEnabled)
@@ -501,6 +512,9 @@ export class WebAR {
         session.requestReferenceSpace('viewer').then((referenceSpace) => {
             session.requestHitTestSource({ space: referenceSpace }).then((source) => {
                 this.hitTestSource = source;
+            }).catch((_) => {
+                this.noHitTestAvailable = true;
+                console.warn("WebXR: Hit test not supported");
             });
         });
 
@@ -533,7 +547,7 @@ export class WebAR {
         if (this.arDomOverlay && this.arOverlayElement) {
             this.arDomOverlay.onEnterAR(session, this.arOverlayElement);
         }
-        
+
         this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
     }
 
@@ -551,11 +565,29 @@ export class WebAR {
             this.sessionRoot.onEnd(this.webxr.Rig, _session);
         }
         if (this.arDomOverlay) this.arDomOverlay.onExitAR(_session);
-        
+
         this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
     }
 
     onUpdate(session: THREE.XRSession, frame: THREE.XRFrame) {
+
+        if (this.noHitTestAvailable === true) {
+            if (this.reticle)
+                this.reticle.visible = false;
+            if (!this.didPlaceARSessionRoot) {
+                this.didPlaceARSessionRoot = true;
+                const rig = this.webxr.Rig;
+                const placementMatrix = arPlacementWithoutHitTestMatrix.clone();
+                if (rig) {
+                    const positionFromRig = new Vector3(0, 0, 0).add(rig.position).divideScalar(this.sessionRoot?.arScale ?? 1);
+                    placementMatrix.multiply(new Matrix4().makeTranslation(positionFromRig.x, positionFromRig.y, positionFromRig.z));
+                    // placementMatrix.setPosition(positionFromRig);
+                }
+                this.sessionRoot?.placeAt(rig, placementMatrix);
+            }
+            return;
+        }
+
         if (!this.hitTestSource) return;
         const hitTestResults = frame.getHitTestResults(this.hitTestSource);
         if (hitTestResults.length) {
@@ -563,7 +595,11 @@ export class WebAR {
             const referenceSpace = this.webxr.context.renderer.xr.getReferenceSpace();
             if (referenceSpace) {
                 const pose = hit.getPose(referenceSpace);
-                this.sessionRoot?.onUpdate(this.webxr.Rig, session, pose);
+
+                if (this.sessionRoot) {
+                    const didPlace = this.sessionRoot.onUpdate(this.webxr.Rig, session, pose);
+                    this.didPlaceARSessionRoot = didPlace;
+                }
 
                 if (this.reticle) {
                     this.reticle.visible = this.reticleActive;
@@ -586,4 +622,4 @@ export class WebAR {
     }
 }
 
-
+const arPlacementWithoutHitTestMatrix = new THREE.Matrix4().identity().makeTranslation(0, -0.5, 0);

package/src/engine-components/WebXRAvatar.ts

@@ -10,6 +10,8 @@ import { Object3D } from "three";
 import { VRUserState } from "./WebXRSync";
 import { getParam } from "../engine/engine_utils";
 import { serializeable } from "../engine/engine_serialization_decorator";
+import { ViewDevice } from "../engine/engine_playerview";
+import { InstancingUtil } from "./Renderer";
 
 export const debug = getParam("debugavatar");
 
@@ -18,8 +20,14 @@ export type AvatarMarkerEventArgs = {
     gameObject: Object3D;
 }
 
-
 export class AvatarMarker extends Behaviour {
+
+    public static getAvatar(index: number): AvatarMarker | null {
+        if (index >= 0 && index < AvatarMarker.instances.length)
+            return AvatarMarker.instances[index];
+        return null;
+    }
+
     public static instances: AvatarMarker[] = [];
 
     public static onAvatarMarkerCreated(cb: (args: AvatarMarkerEventArgs) => void): Function {
@@ -55,7 +63,7 @@ export class AvatarMarker extends Behaviour {
             cb({ avatarMarker: this, gameObject: this.gameObject });
     }
 
-    isLocalAvatar(){
+    isLocalAvatar() {
         return this.connectionId === this.context.connection.connectionId;
     }
 
@@ -80,7 +88,10 @@ export class WebXRAvatar {
         this.updateVisibility();
     }
 
+    get isWebXRAvatar() { return true; }
+
     // TODO: set layers on all avatars 
+    /** the user id */
     public guid: string;
 
     private root: THREE.Object3D | null = null;
@@ -148,6 +159,18 @@ export class WebXRAvatar {
 
             this.lastUpdate = state.time;
             if (this.head) {
+
+                const device = this.webxr.IsInAR ? ViewDevice.Handheld : ViewDevice.Headset;
+                let viewObj = this.head;
+                // if (this.isLocalAvatar) {
+                //     if (this.context.mainCamera && this.context.isInXR) {
+                //         viewObj = this.context.renderer.xr.getCamera(this.context.mainCamera);
+                //     }
+                // }
+                this.context.players.setPlayerView(state.guid, viewObj, device);
+
+                InstancingUtil.markDirty(this.head);
+
                 this._canInterpolate = true;
                 const ht = this.isLocalAvatar ? this.head : this._headTarget;
                 ht.position.set(state.position.x, state.position.y, state.position.z);
@@ -163,6 +186,7 @@ export class WebXRAvatar {
                     ht.quaternion.multiply(WebXRAvatar.invertRotation);
                     ht.scale.set(state.scale, state.scale, state.scale);
                     ht.scale.multiply(this.handLeftScale);
+                    InstancingUtil.markDirty(this.handLeft);
                 }
 
                 if (this.handRight) {
@@ -172,6 +196,7 @@ export class WebXRAvatar {
                     ht.quaternion.multiply(WebXRAvatar.invertRotation);
                     ht.scale.set(state.scale, state.scale, state.scale);
                     ht.scale.multiply(this.handRightScale);
+                    InstancingUtil.markDirty(this.handRight);
                 }
             }
         }

package/src/engine-components/WebXRController.ts

@@ -13,6 +13,9 @@ import * as utils from "../engine/engine_three_utils"
 import { Interactable, UsageMarker } from "./Interactable";
 import { InstancingUtil } from "./Renderer";
 import { Rigidbody } from "./RigidBody";
+import { getParam } from "../engine/engine_utils";
+
+const debug = getParam("debugwebxrcontroller");
 
 export enum ControllerType {
     PhysicalDevice = 0,
@@ -103,10 +106,10 @@ export class WebXRController extends Behaviour {
                 // this is a workaround to fix that temporarely
                 // see https://github.com/needle-tools/needle-tiny-playground/issues/123
                 const jnts = ctrl.hand["joints"];
-                if(jnts){
-                    for(const key of Object.keys(jnts)){
+                if (jnts) {
+                    for (const key of Object.keys(jnts)) {
                         const joint = jnts[key];
-                        if(joint.parent) continue;
+                        if (joint.parent) continue;
                         ctrl.hand.add(joint);
                     }
                 }
@@ -189,7 +192,7 @@ export class WebXRController extends Behaviour {
     private _useSmoothing: boolean = true;
 
     awake(): void {
-        if(!this.controller){
+        if (!this.controller) {
             console.warn("Missing Controller!!!", this);
             return;
         }
@@ -251,7 +254,8 @@ export class WebXRController extends Behaviour {
         this.raycastLine?.add(this._raycastHitPoint);
         this._raycastHitPoint.visible = false;
         this.hand.add(this.handPointerModel);
-        console.log("ADDED TO RIG", this.webXR.Rig);
+        if (debug)
+            console.log("ADDED TO RIG", this.webXR.Rig);
 
         // // console.log("enable", this.index, this.controllerGrip.uuid)
     }
@@ -427,7 +431,8 @@ export class WebXRController extends Behaviour {
                 this.movementVector.applyQuaternion(this.webXR.TransformOrientation);
                 this.movementVector.y = 0;
                 this.movementVector.applyQuaternion(this.worldRot);
-                rig.position.add(this.movementVector.multiplyScalar(speedFactor * this.context.time.deltaTime));
+                this.movementVector.multiplyScalar(speedFactor * this.context.time.deltaTime);
+                rig.position.add(this.movementVector);
 
                 if (this.isUsingHands)
                     this.runTeleport(rig, buttons);
@@ -472,7 +477,7 @@ export class WebXRController extends Behaviour {
         }
         else this._pinchStartTime = undefined;
 
-        let doTeleport = teleport > .5;
+        let doTeleport = teleport > .5 && this.webXR.IsInVR;
         let isInMiniatureMode = this.webXR.Rig ? this.webXR.Rig?.scale?.x < .999 : false;
         let newRigScale: number | null = null;
 
@@ -482,7 +487,7 @@ export class WebXRController extends Behaviour {
                 // button[4] seems to be the A button if it exists. On hololens it's randomly pressed though for hands
                 // see https://www.w3.org/TR/webxr-gamepads-module-1/#xr-standard-gamepad-mapping
                 if (i === 4) {
-                    if (btn.pressed && !this.didChangeScale) {
+                    if (btn.pressed && !this.didChangeScale && this.webXR.IsInVR) {
                         this.didChangeScale = true;
                         const rig = this.webXR.Rig;
                         if (rig) {
@@ -566,7 +571,7 @@ export class WebXRController extends Behaviour {
                 const scale = Mathf.clamp(this.lastHit.distance * .01 * factor, .015, .1);
                 this._raycastHitPoint.scale.set(scale, scale, scale);
             }
-            this._raycastHitPoint.visible = this.lastHit !== null;
+            this._raycastHitPoint.visible = this.lastHit !== null && this.lastHit !== undefined;
         }
         return hit;
     }
@@ -583,7 +588,7 @@ export class WebXRController extends Behaviour {
     }
 
     private selectStartCallback: Function | null = null;
-    private lastSelectStartObject : THREE.Object3D | null = null;;
+    private lastSelectStartObject: THREE.Object3D | null = null;;
 
     private onHandleSelectStart() {
         this.selectStartCallback = null;
@@ -611,7 +616,8 @@ export class WebXRController extends Behaviour {
             for (const intersection of intersections) {
                 const object = intersection.object;
                 if (!this.testIsVisible(object)) {
-                    console.log("not visible");
+                    if (debug)
+                        console.log("not visible");
                     continue;
                 }
                 this.lastSelectStartObject = object;
@@ -622,7 +628,7 @@ export class WebXRController extends Behaviour {
                         sub(this, args);
                     }
                 }
-                if (args.grab !== object)
+                if (args.grab !== object && debug)
                     console.log("Grabbed object changed", "original", object, "new", args.grab);
                 if (args.grab) {
                     this.grabbed = AttachedObject.TryTake(this, args.grab, intersection, closeGrab);
@@ -630,8 +636,7 @@ export class WebXRController extends Behaviour {
                 break;
             }
         }
-        else 
-        {
+        else {
             const subs = WebXRController.eventSubs[ControllerEvents.SelectStart];
             const args = { selected: null, grab: null };
             if (subs && subs.length > 0) {
@@ -761,7 +766,8 @@ export class AttachedObject {
     public static TryTake(controller: WebXRController, candidate: THREE.Object3D, intersection: THREE.Intersection, closeGrab: boolean): AttachedObject | null {
         const interactable = GameObject.getComponentInParent(candidate, Interactable);
         if (!interactable) {
-            console.warn("Prevented taking object that is not interactable", candidate);
+            if (debug)
+                console.warn("Prevented taking object that is not interactable", candidate);
             return null;
         }
 

package/src/engine-components/WebXRSync.ts

@@ -346,8 +346,8 @@ export class WebXRSync extends Behaviour {
 
     private buildLocalAvatar() {
         if (this.localAvatar) return;
-        const id = this.context.connection?.connectionId ?? this.k_LocalAvatarNoNetworkingGuid;
-        this.localAvatar = new WebXRAvatar(this.context, id, this.webXR!);
+        const connectionId = this.context.connection?.connectionId ?? this.k_LocalAvatarNoNetworkingGuid;
+        this.localAvatar = new WebXRAvatar(this.context, connectionId, this.webXR!);
         this.localAvatar.isLocalAvatar = true;
         this.localAvatar.setAvatarOverride(this.getAvatarId());
         this.avatars[this.localAvatar.guid] = this.localAvatar;
@@ -452,7 +452,7 @@ export class WebXRSync extends Behaviour {
 
     private getAvatarId() {
         const urlAvatar = utils.getParam("avatar") as string;
-        const avatarId = urlAvatar;
+        const avatarId = urlAvatar ?? null;
         return avatarId;
     }
 }

package/src/engine-components/XRFlag.ts

@@ -11,6 +11,7 @@ export enum XRStateFlag {
     VR = 1 << 2,
     FirstPerson = 1 << 3,
     ThirdPerson = 1 << 4,
+    All = 0xffffffff
 }
 
 // console.log(XRStateFlag);

package/src/engine-components/ui/CanvasGroup.ts

@@ -16,6 +16,8 @@ export class CanvasGroup extends Behaviour implements ICanvasGroup {
         this.markDirty();
     }
 
+    get isCanvasGroup() { return true; }
+
     private _alpha: number = 1;
 
     @serializeable()

package/src/engine-components/ui/EventSystem.ts

@@ -9,9 +9,8 @@ import { PointerEventData } from "./PointerEvents";
 import { Raycaster } from "./Raycaster";
 import { InputEvents, KeyCode } from "../../engine/engine_input";
 import { Object3D } from "three";
-import { ICanvasGroup } from "./Interfaces";
+import { ICanvasGroup, IGraphic } from "./Interfaces";
 import { getParam } from "../../engine/engine_utils";
-import { off } from "process";
 
 const debug = getParam("debugeventsystem");
 
@@ -135,7 +134,7 @@ export class EventSystem extends Behaviour {
     private objectsHoveredLastFrame: THREE.Object3D[] = [];
     // todo: these should be listed by input source (e.g. per controller, mouse, touch)
     private raisedPointerDownEvents: any[] = [];
-    private _didMove : boolean = false;
+    private _didMove: boolean = false;
 
     onBeforeUpdate() {
         this.objectsHoveredThisFrame.length = 0;
@@ -147,9 +146,9 @@ export class EventSystem extends Behaviour {
             // console.log("alt pressed");
             return;
         }
-        if(!this._didMove){
+        if (!this._didMove) {
             const pos = this.context.input.getPointerPositionRC(0);
-            if(pos && pos.x === 0 && pos.y === 0) return;
+            if (pos && pos.x === 0 && pos.y === 0) return;
             this._didMove = true;
         }
 
@@ -166,10 +165,10 @@ export class EventSystem extends Behaviour {
         // handle orbit control before handling input events
         // this is just so button events that modify OrbitControls.enabled work
         let orbitControlComponent: OrbitControls | null = null;
-        let previousOrbitControlState : boolean | null = null;
+        let previousOrbitControlState: boolean | null = null;
         if (this.context.input.mouseDown && this.currentActiveMeshUIComponents.length > 0 && this.context.mainCameraComponent) {
             orbitControlComponent = GameObject.getComponent(this.context.mainCameraComponent.gameObject, OrbitControls) ?? null;
-            if(orbitControlComponent){
+            if (orbitControlComponent) {
                 previousOrbitControlState = orbitControlComponent.enabled;
                 orbitControlComponent.enabled = false;
             }
@@ -274,16 +273,16 @@ export class EventSystem extends Behaviour {
         this._noDepthTestingResults.length = 0;
         for (let i = 0; i < hits.length; i++) {
             const hit = hits[i];
-            const object = hit.object as THREE.Mesh;   
-            if(object.material){
-                if(object.material["depthTest"] === false){
+            const object = hit.object as THREE.Mesh;
+            if (object.material) {
+                if (object.material["depthTest"] === false) {
                     this._noDepthTestingResults.push(hit);
                     continue;
                 }
             }
             this._sortingBuffer.push(hit);
         }
-        for(const obj of this._sortingBuffer){
+        for (const obj of this._sortingBuffer) {
             this._noDepthTestingResults.push(obj);
         }
         return this._noDepthTestingResults;
@@ -294,7 +293,7 @@ export class EventSystem extends Behaviour {
 
         if (!this.testIsVisible(object)) {
             if (args.isClicked && debug)
-                console.log("not allowed", object);        
+                console.log("not allowed", object);
             return false;
         }
 
@@ -314,10 +313,17 @@ export class EventSystem extends Behaviour {
                 const actualGo = parent.shadowComponentOwner.gameObject;
                 if (actualGo) {
                     canvasGroup = this.tryFindCanvasGroup(actualGo);
-                    if (canvasGroup?.blocksRaycasts === false) return false;
-                    if (canvasGroup?.interactable === false) {
-                        return true;
+                    if (canvasGroup?.isCanvasGroup === true) {
+                        if (canvasGroup.blocksRaycasts === false) return false;
+                        if (canvasGroup.interactable === false) return true;
                     }
+                    // handle Graphic Raycast target
+                    const graphic = GameObject.foreachComponent(actualGo, c => {
+                        if (typeof (c as unknown as IGraphic).raycastTarget === "boolean") return c;
+                        return undefined;
+                    }, false);
+                    if (graphic?.raycastTarget === false) return false;
+
                     const handled = this.handleMeshUIIntersection(object, pressedOrClicked);
                     if (!clicked && handled) {
                         // return true;

package/src/engine-components/ui/Graphic.ts

@@ -10,6 +10,8 @@ import { onChange, scheduleAction } from "./Utils"
 
 export class Graphic extends BaseUIComponent implements IGraphic {
 
+    get isGraphic() { return true; }
+
     @serializeable(RGBAColor)
     get color(): RGBAColor {
         if (!this._color) this._color = new RGBAColor(1, 1, 1, 1);
@@ -132,6 +134,7 @@ export class Graphic extends BaseUIComponent implements IGraphic {
     }
 
     protected onAfterAddedToScene(): void {
+        super.onAfterAddedToScene();
         if (this.shadowComponent) {
             //@ts-ignore
             this.shadowComponent.offset = this.shadowComponent.position.z;

package/src/engine-components/ui/Interfaces.ts

@@ -1,8 +1,10 @@
 export interface ICanvasGroup {
+    get isCanvasGroup() : boolean;
     blocksRaycasts: boolean;
     interactable: boolean;
 }
 
 export interface IGraphic {
+    get isGraphic() : boolean;
     raycastTarget: boolean;
 }