@needle-tools/engine 2.35.5-pre → 2.37.0-pre

package/src/engine-components/WebARSessionRoot.ts

@@ -1,17 +1,17 @@
 import { Behaviour, GameObject } from "./Component";
-import * as THREE from 'three'
+import { XRSession } from 'three'
 import { Matrix4, Object3D, XRPose } from "three";
 import { WebAR, WebXR } from "./WebXR";
 import { InstancingUtil } from "../engine/engine_instancing";
 import { serializeable } from "../engine/engine_serialization_decorator";
 
-// https://github.com/takahirox/takahirox.github.io/blob/master/three.js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
+// https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js
 
 export class WebARSessionRoot extends Behaviour {
 
     webAR: WebAR | null = null;
 
-    get rig(): THREE.Object3D | undefined {
+    get rig(): Object3D | undefined {
         return this.webAR?.webxr.Rig;
     }
 
@@ -28,7 +28,7 @@ export class WebARSessionRoot extends Behaviour {
         this.setScale(val);
     }
 
-    private readonly _initalMatrix = new THREE.Matrix4();
+    private readonly _initalMatrix = new Matrix4();
     private readonly _selectStartFn = this.onSelectStart.bind(this);
     private readonly _selectEndFn = this.onSelectEnd.bind(this);
 
@@ -41,13 +41,13 @@ export class WebARSessionRoot extends Behaviour {
     }
 
     private _arScale: number = 5;
-    private _rig: THREE.Object3D | null = null;
-    private _startPose: THREE.Matrix4 | null = null;
-    private _placementPose: THREE.Matrix4 | null = null;
+    private _rig: Object3D | null = null;
+    private _startPose: Matrix4 | null = null;
+    private _placementPose: Matrix4 | null = null;
     private _isTouching: boolean = false;
-    private _rigStartPose: THREE.Matrix4 | undefined | null = null;
+    private _rigStartPose: Matrix4 | undefined | null = null;
 
-    onBegin(session: THREE.XRSession) {
+    onBegin(session: XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
@@ -69,12 +69,12 @@ export class WebARSessionRoot extends Behaviour {
         }
     }
 
-    onUpdate(rig: THREE.Object3D | null, _session: THREE.XRSession, pose: XRPose | null | undefined): boolean {
+    onUpdate(rig: Object3D | null, _session: XRSession, pose: XRPose | null | undefined): boolean {
 
         if (pose && !this._placementPose) {
             if (this._isTouching) {
                 if (this.webAR) this.webAR.setReticleActive(false);
-                this.placeAt(rig, new THREE.Matrix4().fromArray(pose.transform.matrix).invert());
+                this.placeAt(rig, new Matrix4().fromArray(pose.transform.matrix).invert());
                 return true;
             }
         }
@@ -90,13 +90,13 @@ export class WebARSessionRoot extends Behaviour {
         // }
     }
 
-    placeAt(rig: THREE.Object3D | null, mat: Matrix4) {
-        if (!this._placementPose) this._placementPose = new THREE.Matrix4();
+    placeAt(rig: Object3D | null, mat: Matrix4) {
+        if (!this._placementPose) this._placementPose = new Matrix4();
         this._placementPose.copy(mat);
         if (rig) {
 
             if (this.invertForward) {
-                const rot = new THREE.Matrix4().makeRotationY(Math.PI);
+                const rot = new Matrix4().makeRotationY(Math.PI);
                 this._placementPose.premultiply(rot);
             }
             this._rig = rig;
@@ -111,7 +111,7 @@ export class WebARSessionRoot extends Behaviour {
         this.gameObject.visible = true;
     }
 
-    onEnd(rig: THREE.Object3D | null, _session: THREE.XRSession) {
+    onEnd(rig: Object3D | null, _session: XRSession) {
         this._placementPose = null;
         this.gameObject.visible = false;
         this.gameObject.matrixAutoUpdate = false;
@@ -151,7 +151,7 @@ export class WebARSessionRoot extends Behaviour {
             this._rigStartPose = rig.matrix.clone();
         }
         rig.matrixAutoUpdate = false;
-        rig.matrix.multiplyMatrices(new THREE.Matrix4().makeScale(scale, scale, scale), this._placementPose);
+        rig.matrix.multiplyMatrices(new Matrix4().makeScale(scale, scale, scale), this._placementPose);
         rig.matrix.decompose(rig.position, rig.quaternion, rig.scale);
         rig.updateMatrixWorld();
         console.log("Place", rig.position);

package/src/engine-components/WebXR.ts

@@ -1,23 +1,21 @@
-import { Behaviour, GameObject } from "./Component";
-
+import { ArrayCamera, Color, Euler, EventDispatcher, Group, Matrix4, Mesh, MeshBasicMaterial, Object3D, Quaternion, RingGeometry, Texture, Vector3, XRFrame, XRHitTestSource, XRRigidTransform, XRSession, XRViewerPose } from 'three';
 import { ARButton } from '../include/three/ARButton.js';
 import { VRButton } from '../include/three/VRButton.js';
 
-import * as THREE from 'three'
-import { ArrayCamera, Matrix4, Vector3, XRHitTestSource } from 'three';
+import { AssetReference } from "../engine/engine_addressables";
+import { serializeable } from "../engine/engine_serialization_decorator";
+import { XRSessionMode } from "../engine/engine_setup";
+import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
+import { INeedleEngineComponent } from "../engine/engine_types";
+import { getParam, setOrAddParamsToUrl } from "../engine/engine_utils";
 
-import * as utils from "../engine/engine_utils";
+import { Behaviour, GameObject } from "./Component";
 import { noVoip } from "./Voip";
-import { XRFlag, XRState, XRStateFlag } from "./XRFlag";
-import { ControllerType, WebXRController } from "./WebXRController";
 import { WebARSessionRoot } from "./WebARSessionRoot";
-import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
+import { ControllerType, WebXRController } from "./WebXRController";
 import { XRRig } from "./WebXRRig";
-import { AssetReference } from "../engine/engine_addressables";
-import { serializeable } from "../engine/engine_serialization_decorator";
 import { WebXRSync } from "./WebXRSync";
-import { XRSessionMode } from "../engine/engine_setup";
-import { INeedleEngineComponent } from "../engine/engine_types";
+import { XRFlag, XRState, XRStateFlag } from "./XRFlag";
 
 
 export async function detectARSupport() {
@@ -38,7 +36,6 @@ let vrSupported = false;
 detectARSupport().then(res => arSupported = res);
 detectVRSupport().then(res => vrSupported = res);
 
-
 // import TeleportVR from "teleportvr";
 
 export enum WebXREvent {
@@ -54,16 +51,23 @@ export declare type CreateButtonOptions = {
 
 export class WebXR extends Behaviour {
 
+    @serializeable()
     enableVR = true;
+    @serializeable()
     enableAR = true;
+
     @serializeable(AssetReference)
     defaultAvatar?: AssetReference;
+    @serializeable()
+    handModelPath: string = "";
 
+    @serializeable()
     createVRButton: boolean = true;
+    @serializeable()
     createARButton: boolean = true;
 
     private static _isInXr: boolean = false;
-    private static events: THREE.EventDispatcher = new THREE.EventDispatcher();
+    private static events: EventDispatcher = new EventDispatcher();
 
     public static get IsInWebXR(): boolean { return this._isInXr; }
     public static get XRSupported(): boolean { return 'xr' in navigator && (arSupported || vrSupported); }
@@ -121,7 +125,7 @@ export class WebXR extends Behaviour {
         if (session) session.end();
     }
 
-    public get Rig(): THREE.Object3D {
+    public get Rig(): Object3D {
         if(!this.rig) this.ensureRig();
         return this.rig;
     }
@@ -154,7 +158,7 @@ export class WebXR extends Behaviour {
     public get IsInVR() { return this._isInVR; }
     public get IsInAR() { return this._isInAR; }
 
-    private rig!: THREE.Object3D;
+    private rig!: Object3D;
     private isInit: boolean = false;
 
     private _requestedAR: boolean = false;
@@ -221,11 +225,11 @@ export class WebXR extends Behaviour {
         }, 1000);
     }
 
-    private _transformOrientation: THREE.Quaternion = new THREE.Quaternion();
-    public get TransformOrientation(): THREE.Quaternion { return this._transformOrientation; }
+    private _transformOrientation: Quaternion = new Quaternion();
+    public get TransformOrientation(): Quaternion { return this._transformOrientation; }
 
-    private _currentHeadPose: THREE.XRViewerPose | null = null;
-    public get HeadPose(): THREE.XRViewerPose | null { return this._currentHeadPose; }
+    private _currentHeadPose: XRViewerPose | null = null;
+    public get HeadPose(): XRViewerPose | null { return this._currentHeadPose; }
 
     onBeforeRender(frame) {
         if (!frame) return;
@@ -237,7 +241,7 @@ export class WebXR extends Behaviour {
             const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace());
             this._currentHeadPose = pose;
             if (!pose) return; // e.g. if user is not wearing headset
-            const transform: THREE.XRRigidTransform = pose?.transform;
+            const transform: XRRigidTransform = pose?.transform;
 
             if (transform) {
                 this._transformOrientation.set(transform.orientation.x, transform.orientation.y, transform.orientation.z, transform.orientation.w);
@@ -312,7 +316,7 @@ export class WebXR extends Behaviour {
                 // this.rig.quaternion.premultiply(existing.worldQuaternion);
             }
             else {
-                this.rig = new THREE.Group();
+                this.rig = new Group();
                 this.rig.rotateY(Math.PI);
                 this.rig.name = "XRRig";
                 this.context.scene.add(this.rig);
@@ -321,11 +325,11 @@ export class WebXR extends Behaviour {
     }
 
 
-    private _originalCameraParent: THREE.Object3D | null = null;
-    private _originalCameraPosition: THREE.Vector3 = new THREE.Vector3();
-    private _originalCameraRotation: THREE.Quaternion = new THREE.Quaternion();
+    private _originalCameraParent: Object3D | null = null;
+    private _originalCameraPosition: Vector3 = new Vector3();
+    private _originalCameraRotation: Quaternion = new Quaternion();
 
-    private onEnterXR(session: THREE.XRSession, frame: THREE.XRFrame) {
+    private onEnterXR(session: XRSession, frame: XRFrame) {
         console.log("[XR] session begin", session);
         WebXR._isInXr = true;
 
@@ -336,8 +340,8 @@ export class WebXR extends Behaviour {
             const pose = frame.getViewerPose(space);
             const rot = pose?.transform.orientation;
             if (rot) {
-                const quat = new THREE.Quaternion(rot.x, rot.y, rot.z, rot.w);
-                const eu = new THREE.Euler().setFromQuaternion(quat);
+                const quat = new Quaternion(rot.x, rot.y, rot.z, rot.w);
+                const eu = new Euler().setFromQuaternion(quat);
                 this.rig.rotateY(eu.y);
                 // this.rig.quaternion.multiply(quat);
             }
@@ -383,7 +387,7 @@ export class WebXR extends Behaviour {
         WebXR.events.dispatchEvent({ type: WebXREvent.XRStarted, session: session });
     }
 
-    private onExitXR(session: THREE.XRSession | null) {
+    private onExitXR(session: XRSession | null) {
 
         if (this._isInAR && session) {
             this.webAR?.onEnd(session);
@@ -413,7 +417,7 @@ export class WebXR extends Behaviour {
         WebXR.events.dispatchEvent({ type: WebXREvent.XRStopped, session: session });
     }
 
-    private onEnterVR(_session: THREE.XRSession) {
+    private onEnterVR(_session: XRSession) {
     }
 
     private destroyControllers() {
@@ -425,13 +429,13 @@ export class WebXR extends Behaviour {
 
     private xrMirrorWindow: Window | null = null;
 
-    private onEnterXR_HandleMirrorWindow(session: THREE.XRSession) {
-        if (!utils.getParam("mirror")) return;
+    private onEnterXR_HandleMirrorWindow(session: XRSession) {
+        if (!getParam("mirror")) return;
         setTimeout(() => {
             if (!WebXR.IsInWebXR) return;
             const url = new URL(window.location.href);
-            utils.setOrAddParamsToUrl(url.searchParams, noVoip, 1);
-            utils.setOrAddParamsToUrl(url.searchParams, "isMirror", 1);
+            setOrAddParamsToUrl(url.searchParams, noVoip, 1);
+            setOrAddParamsToUrl(url.searchParams, "isMirror", 1);
             const str = url.toString();
             this.xrMirrorWindow = window.open(str, "webxr sync", "popup=yes");
             if (this.xrMirrorWindow) {
@@ -456,20 +460,21 @@ export class WebAR {
 
     private _webxr: WebXR;
 
-    private reticle: THREE.Object3D | null = null;
+    private reticle: Object3D | null = null;
+    private reticleParent: Object3D | null = null;
     private hitTestSource: XRHitTestSource | null = null;
     private reticleActive: boolean = true;
 
     // scene.background before entering AR
-    private previousBackground: THREE.Color | null | THREE.Texture = null;
-    private previousEnvironment: THREE.Texture | null = null;
+    private previousBackground: Color | null | Texture = null;
+    private previousEnvironment: Texture | null = null;
 
     private sessionRoot: WebARSessionRoot | null = null;
-    private _previousParent: THREE.Object3D | null = null;
+    private _previousParent: Object3D | null = null;
     // we need this in case the session root is on the same object as the webxr component
     // so if we disable the session root we attach the webxr component to this temporary object
     // to still receive updates
-    private static tempWebXRObject: THREE.Object3D;
+    private static tempWebXRObject: Object3D;
 
     private get context() { return this.webxr.context; }
 
@@ -496,13 +501,14 @@ export class WebAR {
         this.reticleActive = active;
     }
 
-    async onBegin(session: THREE.XRSession) {
+    async onBegin(session: XRSession) {
         const context = this.webxr.context;
         this.reticleActive = true;
         this.didPlaceARSessionRoot = false;
 
         const deviceType = navigator.userAgent?.includes("OculusBrowser") ? ControllerType.PhysicalDevice : ControllerType.Touch;
-        for (let i = 0; i < 4; i++) {
+        const controllerCount = deviceType === ControllerType.Touch ? 4 : 2;
+        for (let i = 0; i < controllerCount; i++) {
             WebXRController.Create(this.webxr, i, this.webxr.gameObject as GameObject, deviceType)
         }
 
@@ -523,22 +529,31 @@ export class WebAR {
         });
 
         if (!this.reticle && this.sessionRoot) {
-            this.reticle = new THREE.Mesh(
-                new THREE.RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
-                new THREE.MeshBasicMaterial()
+            this.reticle = new Mesh(
+                new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2),
+                new MeshBasicMaterial()
             );
             this.reticle.name = "AR Placement reticle";
             this.reticle.matrixAutoUpdate = false;
             this.reticle.visible = false;
+
+            // create AR reticle parent to allow WebXRSessionRoot to be translated, rotated or scaled
+            this.reticleParent = new Object3D();
+            this.reticleParent.name = "AR Reticle Parent";
+            this.reticleParent.matrixAutoUpdate = false;
+            this.reticleParent.add(this.reticle);
+            this.reticleParent.matrix.copy(this.sessionRoot.gameObject.matrixWorld);
+            
             if (this.webxr.scene) {
-                this.webxr.scene.add(this.reticle);
-                this.webxr.scene.visible = true;
+                this.context.scene.add(this.reticleParent);
+                // this.context.scene.add(this.reticle);
+                this.context.scene.visible = true;
             }
             else console.warn("Could not found WebXR Rig");
         }
 
         this._previousParent = this.webxr.gameObject;
-        if (!WebAR.tempWebXRObject) WebAR.tempWebXRObject = new THREE.Object3D();
+        if (!WebAR.tempWebXRObject) WebAR.tempWebXRObject = new Object3D();
         this.context.scene.add(WebAR.tempWebXRObject);
         GameObject.addComponent(WebAR.tempWebXRObject as GameObject, this.webxr);
 
@@ -556,7 +571,7 @@ export class WebAR {
         this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
     }
 
-    onEnd(session: THREE.XRSession) {
+    onEnd(session: XRSession) {
         if (this._previousParent) {
             console.log("ADD", this._previousParent);
             GameObject.addComponent(this._previousParent as GameObject, this.webxr);
@@ -578,7 +593,7 @@ export class WebAR {
         this.context.mainCameraComponent?.applyClearFlagsIfIsActiveCamera();
     }
 
-    onUpdate(session: THREE.XRSession, frame: THREE.XRFrame) {
+    onUpdate(session: XRSession, frame: XRFrame) {
 
         if (this.noHitTestAvailable === true) {
             if (this.reticle)
@@ -631,4 +646,4 @@ export class WebAR {
     }
 }
 
-const arPlacementWithoutHitTestMatrix = new THREE.Matrix4().identity().makeTranslation(0, -0.5, 0);
+const arPlacementWithoutHitTestMatrix = new Matrix4().identity().makeTranslation(0, -0.5, 0);

package/src/engine-components/WebXRAvatar.ts

@@ -1,6 +1,6 @@
 import { Behaviour, GameObject } from "./Component";
 import { WebXR } from "./WebXR";
-import * as THREE from "three";
+import { Quaternion, Vector3 } from "three";
 import { AvatarLoader } from "./AvatarLoader";
 import { XRFlag, XRStateFlag } from "./XRFlag";
 import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt";
@@ -45,7 +45,7 @@ export class AvatarMarker extends Behaviour {
 
 
     public connectionId!: string;
-    public avatar?: WebXRAvatar | THREE.Object3D;
+    public avatar?: WebXRAvatar | Object3D;
 
     awake() {
         AvatarMarker.instances.push(this);
@@ -94,16 +94,16 @@ export class WebXRAvatar {
     /** the user id */
     public guid: string;
 
-    private root: THREE.Object3D | null = null;
-    public head: THREE.Object3D | null = null;
-    public handLeft: THREE.Object3D | null = null;
-    public handRight: THREE.Object3D | null = null;
+    private root: Object3D | null = null;
+    public head: Object3D | null = null;
+    public handLeft: Object3D | null = null;
+    public handRight: Object3D | null = null;
     public lastUpdate: number = -1;
     public isLocalAvatar: boolean = false;
     public flags: XRFlag[] | null = null;
-    private headScale: THREE.Vector3 = new THREE.Vector3(1, 1, 1);
-    private handLeftScale: THREE.Vector3 = new THREE.Vector3(1, 1, 1);
-    private handRightScale: THREE.Vector3 = new THREE.Vector3(1, 1, 1);
+    private headScale: Vector3 = new Vector3(1, 1, 1);
+    private handLeftScale: Vector3 = new Vector3(1, 1, 1);
+    private handRightScale: Vector3 = new Vector3(1, 1, 1);
 
     private readonly webxr: WebXR;
 
@@ -142,12 +142,12 @@ export class WebXRAvatar {
         return null;
     }
 
-    private _headTarget: THREE.Object3D = new THREE.Object3D();
-    private _handLeftTarget: THREE.Object3D = new THREE.Object3D();
-    private _handRightTarget: THREE.Object3D = new THREE.Object3D();
+    private _headTarget: Object3D = new Object3D();
+    private _handLeftTarget: Object3D = new Object3D();
+    private _handRightTarget: Object3D = new Object3D();
     private _canInterpolate: boolean = false;
 
-    private static invertRotation: THREE.Quaternion = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);
+    private static invertRotation: Quaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
 
     public tryUpdate(state: VRUserState, _timeDiff: number) {
         if (state.guid === this.guid) {
@@ -247,7 +247,7 @@ export class WebXRAvatar {
         }
     }
 
-    private async setupCustomAvatar(avatarId: string | THREE.Object3D | AssetReference): Promise<boolean> {
+    private async setupCustomAvatar(avatarId: string | Object3D | AssetReference): Promise<boolean> {
         if (debug)
             console.log("LOAD", avatarId, this);
 
@@ -263,14 +263,14 @@ export class WebXRAvatar {
             await reference.loadAssetAsync();
             const prefab = reference.asset as Object3D;
             GameObject.setActive(prefab, false);
-            avatarId = GameObject.instantiate(prefab as Object3D) as THREE.Object3D;
+            avatarId = GameObject.instantiate(prefab as Object3D) as Object3D;
             GameObject.setActive(avatarId, true);
             // console.log("Avatar", avatarId);
         }
         if (debug)
             console.log(avatarId);
 
-        const model = await WebXRAvatar.loader.getOrCreateNewAvatarInstance(this.context, avatarId as (THREE.Object3D | string));
+        const model = await WebXRAvatar.loader.getOrCreateNewAvatarInstance(this.context, avatarId as (Object3D | string));
         if (debug)
             console.log(model, model?.isValid, this.lastAvatarId, avatarId);
         // if (this.lastAvatarId !== avatarId) {