@needle-tools/engine 4.12.0-next.c6c0281 → 4.12.0-next.da19dd0

package/src/engine/xr/NeedleXRSession.ts

@@ -6,6 +6,7 @@ import { Context, FrameEvent } from "../engine_context.js";
 import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
 import { isDestroyed } from "../engine_gameobject.js";
 import { Gizmos } from "../engine_gizmos.js";
+import { Telemetry } from "../engine_license.js";
 import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
 import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
 import type { ICamera, IComponent, INeedleXRSession } from "../engine_types.js";
@@ -82,6 +83,34 @@ function getDOMOverlayElement(domElement: HTMLElement) {
 handleSessionGranted();
 async function handleSessionGranted() {
 
+    // await delay(400);
+
+
+    let defaultMode: XRSessionMode = "immersive-vr";
+
+    try {
+        // In app clips we default to AR
+        if (DeviceUtilities.isNeedleAppClip()) {
+            defaultMode = "immersive-ar";
+        }
+        // Check if VR is even supported, otherwise try AR
+        else if (!(await navigator.xr?.isSessionSupported("immersive-vr"))) {
+            defaultMode = "immersive-ar";
+        }
+        // Check if AR is supported, otherwise we can't do anything
+        if (!(await navigator.xr?.isSessionSupported("immersive-ar")) && defaultMode === "immersive-ar") {
+            // console.warn("[NeedleXRSession:granted] Neither VR nor AR supported, aborting session start.");
+            // showBalloonMessage("NeidleXRSession: Neither VR nor AR supported, aborting session start.");
+            return;
+        }
+    } catch (e) {
+        console.debug("[NeedleXRSession:granted] Error while checking XR support:", e);
+        // showBalloonWarning("NeedleXRSession: Error while checking XR support: " + (e as Error).message);
+        return;
+    }
+
+    // showBalloonMessage("sessiongranted: " + defaultMode);
+
     // TODO: asap session granted doesnt handle the pre-room yet
     if (getParam("debugasap")) {
         let asapSession = globalThis["needle:XRSession"] as XRSession | undefined | Promise<XRSession>;
@@ -94,11 +123,11 @@ async function handleSessionGranted() {
                 enableSpatialConsole(true);
                 const session = await asapSession;
                 if (session) {
-                    const sessionInit = NeedleXRSession.getDefaultSessionInit("immersive-vr");
-                    NeedleXRSession.setSession("immersive-vr", session, sessionInit, cb.context);
+                    const sessionInit = NeedleXRSession.getDefaultSessionInit(defaultMode);
+                    NeedleXRSession.setSession(defaultMode, session, sessionInit, cb.context);
                 }
                 else {
-                    console.error("NeedleXRSession: ASAP session was rejected");
+                    console.error("[NeedleXRSession:granted] ASAP session was rejected");
                 }
                 asapSession = undefined;
             });
@@ -106,6 +135,9 @@ async function handleSessionGranted() {
         }
     }
 
+    // console.log("Attaching sessiongranted handler...", { haveXR: 'xr' in navigator });
+    // setTimeout(() => console.log("Session Granted handler attached.", { haveXR: 'xr' in navigator }), 1000);
+
     if ('xr' in navigator) {
         // WebXRViewer (based on Firefox) has a bug where addEventListener
         // throws a silent exception and aborts execution entirely.
@@ -115,10 +147,8 @@ async function handleSessionGranted() {
         }
 
         navigator.xr?.addEventListener('sessiongranted', async () => {
-            enableSpatialConsole(true);
+            // enableSpatialConsole(true);
 
-            console.log("Received Session Granted...")
-            await delay(100);
 
             const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode") as XRSessionMode;
             const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null;
@@ -126,7 +156,8 @@ async function handleSessionGranted() {
 
             let info: SessionInfo | null = null;
             if (contextIsLoading()) {
-                await TemporaryXRContext.start(lastSessionMode || "immersive-vr", init || NeedleXRSession.getDefaultSessionInit("immersive-vr"));
+                await TemporaryXRContext.start(lastSessionMode || defaultMode, init || NeedleXRSession.getDefaultSessionInit(defaultMode))
+                    .catch(e => console.warn("[NeedleXRSession:granted] TemporaryXRContext start failed:", e));
                 await waitForContextLoadingFinished();
                 info = await TemporaryXRContext.handoff();
             }
@@ -134,17 +165,19 @@ async function handleSessionGranted() {
                 NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
             }
             else if (lastSessionMode && lastSessionInit) {
-                console.log("Session Granted: Restore last session")
+                console.log("[NeedleXRSession:granted] Restore last session")
                 const init = JSON.parse(lastSessionInit);
                 NeedleXRSession.start(lastSessionMode as XRSessionMode, init).catch(e => console.warn(e));
             }
             else {
                 // if no session was found we start VR by default
-                NeedleXRSession.start("immersive-vr").catch(e => console.warn("Session Granted failed:", e));
+                NeedleXRSession.start(defaultMode).catch(e => console.warn("[NeedleXRSession:granted] failed:", e));
             }
             // make sure we only subscribe to the event once
         }, { once: true });
-
+    }
+    else {
+        // showBalloonWarning("NeedleXRSession: WebXR not available in this browser.");
     }
 }
 function saveSessionInfo(mode: XRSessionMode, init: XRSessionInit) {
@@ -419,27 +452,78 @@ export class NeedleXRSession implements INeedleXRSession {
      * @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit
      * @param context The Needle Engine context to use
      */
-    static async start(mode: XRSessionMode | "ar", init?: XRSessionInit, context?: Context): Promise<NeedleXRSession | null> {
+    static async start(mode: XRSessionMode | "ar" | "quicklook", init?: XRSessionInit, context?: Context): Promise<NeedleXRSession | null> {
 
-        // handle iOS platform where "immersive-ar" is not supported
+        // handle iOS platform where "immersive-ar" is special:
+        // - we either launch QuickLook
+        // - or forward to the Needle App Clip experience for WebXR AR
         // TODO: should we add a separate mode (e.g. "AR")? https://linear.app/needle/issue/NE-5303
         if (DeviceUtilities.isiOS()) {
-            if (mode === "ar") {
-                const arSupported = await this.isARSupported();
-                if (!arSupported) {
-                    InternalUSDZRegistry.exportAndOpen();
-                    return null;
+
+            const arSupported = await this.isARSupported().catch(() => false);
+
+            // On VisionOS, we use QuickLook for AR experiences; no AppClip support for now.
+            if (DeviceUtilities.isVisionOS() && !arSupported && (mode === "ar" || mode === "immersive-ar")) {
+                mode = "quicklook";
+            }
+
+            if (mode === "quicklook") {
+                Telemetry.sendEvent(Context.Current, "xr", {
+                    action: "quicklook_export",
+                    source: "NeedleXRSession.start",
+                });
+                InternalUSDZRegistry.exportAndOpen();
+                return null;
+            }
+
+            if (!arSupported && (mode === "immersive-ar" || mode === "ar")) {
+                // const debugAppClip = getParam("debugappclip")
+                // Forward to the AppClip experience (Using the apple.com url the appclip overlay shows immediately)
+                // const url =`https://appclip.needle.tools/ar?url=${(location.href)}`;
+                const url = new URL("https://appclip.apple.com/id?p=tools.needle.launch-app.Clip");
+                url.searchParams.set("url", location.href);
+
+                const urlStr = url.toString();
+
+                Telemetry.sendEvent(Context.Current, "xr", {
+                    action: "app_clip_launch",
+                    source: "NeedleXRSession.start",
+                    url: urlStr,
+                });
+
+                // if we are in an iframe, we need to navigate the top window
+                const topWindow = window.top || window;
+                try {
+                    console.debug("iOS device detected - opening Needle App Clip for AR experience", { mode, init, url });
+                    // navigate to app clip url but keep the current url in history, open in same tab
+                    // eslint-disable-next-line xss/no-location-href-assign
+                    topWindow.location.href = urlStr;
                 }
-                else {
-                    mode = "immersive-ar";
+                catch (e) {
+                    console.warn("Error navigating to AppClip " + urlStr + "\n", e);
+                    // if top window navigation fails and we are in an iframe, we try to navigate the top window directly
+                    const weAreInIframe = window !== window.top;
+                    if (weAreInIframe) {
+                        // we can try to open a new tab as a fallback
+                        window.open(urlStr, "_blank");
+                    }
+                    // eslint-disable-next-line xss/no-location-href-assign
+                    else window.location.href = urlStr;
                 }
+
+                return null;
             }
         }
-        else if (mode == "ar") {
-            mode = "immersive-ar";
-        }
 
+        if (mode === "quicklook") {
+            console.warn("QuickLook mode is only supported on iOS devices");
+            return null;
+        }
 
+        // Since we now know we are not on iOS, ar mode becomes "immersive-ar"
+        if (mode == "ar") {
+            mode = "immersive-ar";
+        }
 
         if (isDevEnvironment() && getParam("debugxrpreroom")) {
             console.warn("Debug: Starting temporary XR session");
@@ -535,12 +619,18 @@ export class NeedleXRSession implements INeedleXRSession {
             listener({ mode, init });
         }
         if (debug) showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")");
+        Telemetry.sendEvent(Context.Current, "xr", {
+            action: "session_request",
+            mode: mode,
+            features: ((init.requiredFeatures ?? []).concat(init.optionalFeatures ?? [])).join(","),
+            source: "NeedleXRSession.start",
+        });
         this._currentSessionRequest = navigator?.xr?.requestSession(mode, init);
         this._currentSessionRequestMode = mode;
         /**@type {XRSystem} */
         const newSession = await (this._currentSessionRequest)?.catch(e => {
-            console.error(e, "Code: " + e.code);
-            if (e.code === 9) showBalloonWarning("Make sure your device has the required permissions (e.g. camera access)")
+            console.error(e, "Code: " + e?.code);
+            if (e?.code === 9) showBalloonWarning("Couldn't start XR session. Make sure you allow the required permissions.")
             console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)");
             const notSecure = location.protocol === 'http:';
             if (notSecure) showBalloonWarning("XR requires a secure connection (HTTPS)");
@@ -1111,6 +1201,12 @@ export class NeedleXRSession implements INeedleXRSession {
 
         console.debug("XR Session ended");
 
+        Telemetry.sendEvent(Context.Current, "xr", {
+            action: "session_end",
+            mode: this.mode,
+            source: "NeedleXRSession.onEnd",
+        });
+
         deleteSessionInfo();
 
         this.onAfterRender();

package/src/engine/xr/TempXRContext.ts

@@ -32,7 +32,13 @@ export class TemporaryXRContext {
                 return null;
             }
             this._requestInFlight = true;
-            const session = await navigator.xr.requestSession(mode, init);
+            const session = await navigator.xr.requestSession(mode, init).catch(err => {
+                console.error("Failed to start temporary XR session:", err);
+            });
+            if (!session) {
+                this._requestInFlight = false;
+                return null;
+            }
             session.addEventListener("end", () => {
                 this._active = null;
             });
@@ -82,10 +88,14 @@ export class TemporaryXRContext {
         this._session = session;
         this._session.addEventListener("end", this.onEnd);
 
-        this._renderer = new WebGLRenderer({ alpha: true });
+        this._renderer = new WebGLRenderer({ alpha: true, antialias: true }); 
+        this._renderer.outputColorSpace = 'srgb';
         this._renderer.setAnimationLoop(this.onFrame);
         this._renderer.xr.setSession(session);
         this._renderer.xr.enabled = true;
+        // Set pixel ratio and size
+        this._renderer.setPixelRatio(Math.min(2, window.devicePixelRatio));
+        this._renderer.setSize(window.innerWidth, window.innerHeight);
         this._camera = new PerspectiveCamera();
         this._scene = new Scene();
         this._scene.fog = new Fog(0x444444, 10, 250);

package/src/engine/xr/usdz.ts

@@ -1,3 +1,4 @@
+import { isDevEnvironment } from "../debug/index.js";
 
 declare type USDZExporter = {
     exportAndOpen(): Promise<any>,
@@ -11,7 +12,11 @@ export namespace InternalUSDZRegistry {
     const usdzExporter: USDZExporter[] = [];
 
     export function exportAndOpen(): boolean {
-        if (!usdzExporter?.length) return false;
+        if (!usdzExporter?.length) {
+            if (isDevEnvironment()) {
+                console.warn("No USDZ exporters found – cannot export USDZ for QuickLook.");
+            }
+        }
         for (const exp of usdzExporter) {
             exp.exportAndOpen();
         }

package/src/engine-components/Camera.ts

@@ -8,7 +8,7 @@ import { Context } from "../engine/engine_setup.js";
 import { RenderTexture } from "../engine/engine_texture.js";
 import { getTempColor, getWorldPosition } from "../engine/engine_three_utils.js";
 import type { ICamera } from "../engine/engine_types.js"
-import { getParam } from "../engine/engine_utils.js";
+import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
 import { NeedleXREventArgs } from "../engine/engine_xr.js";
 import { RGBAColor } from "../engine/js-extensions/index.js";
 import { Behaviour, GameObject } from "./Component.js";
@@ -664,6 +664,9 @@ export class Camera extends Behaviour implements ICamera {
                 else if (navigator.userAgent?.includes("Mozilla") && navigator.userAgent?.includes("Mobile WebXRViewer/v2")) {
                     transparent = true;
                 }
+                else if(DeviceUtilities.isNeedleAppClip()) {
+                    return true;
+                }
             }
         }
 

package/src/engine-components/Component.ts

@@ -476,10 +476,11 @@ export abstract class GameObject extends Object3D implements Object3D, IGameObje
      * Gets a component of the specified type in the gameObject's children hierarchy
      * @param go GameObject to search in
      * @param typeName Constructor of the component type
+     * @param includeInactive Whether to include inactive objects in the search
      * @returns The first matching component if found, otherwise null
      */
-    public static getComponentInChildren<T extends IComponent>(go: IGameObject | Object3D, typeName: Constructor<T>): T | null {
-        return getComponentInChildren(go, typeName);
+    public static getComponentInChildren<T extends IComponent>(go: IGameObject | Object3D, typeName: Constructor<T>, includeInactive: boolean = false): T | null {
+        return getComponentInChildren(go, typeName, includeInactive);
     }
 
     /**
@@ -588,7 +589,7 @@ export abstract class Component implements IComponent, EventTarget,
      */
     get [$componentName]() { return TypeStore.getKey(this.constructor as any) || undefined; }
 
-    
+
     private __context: Context | undefined;
 
     /**
@@ -1106,7 +1107,7 @@ export abstract class Component implements IComponent, EventTarget,
             this.dispatchEvent(new CustomEvent("destroyed", { detail: this }));
         }
         destroyComponentInstance(this as any);
-        if(isHotReloadEnabled()) unregisterHotReloadType(this);
+        if (isHotReloadEnabled()) unregisterHotReloadType(this);
     }
 
     /**

package/src/engine-components/DragControls.ts

@@ -447,7 +447,7 @@ export class DragControls extends Behaviour implements IPointerEventHandler {
         if (!this || !this._isDragging) return;
         this._isDragging = false;
         for (const rb of this._draggingRigidbodies) {
-            rb.setVelocity(rb.smoothedVelocity);
+            rb.setVelocity(rb.smoothedVelocity.multiplyScalar(this.context.time.deltaTime));
         }
         this._draggingRigidbodies.length = 0;
         this._targetObject = null;

package/src/engine-components/SpectatorCamera.ts

@@ -8,7 +8,8 @@ import { PlayerView, ViewDevice } from "../engine/engine_playerview.js";
 import { serializable } from "../engine/engine_serialization.js";
 import { Context } from "../engine/engine_setup.js";
 import type { ICamera } from "../engine/engine_types.js";
-import { getParam } from "../engine/engine_utils.js";
+import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
+import { PlayerState } from "../engine-components-experimental/networking/PlayerSync.js";
 import { Camera } from "./Camera.js";
 import { Behaviour, Component, GameObject } from "./Component.js";
 import { OrbitControls } from "./OrbitControls.js";
@@ -98,7 +99,7 @@ export class SpectatorCamera extends Behaviour {
     }
 
     /** Gets the local player's connection ID */
-    private get localId() : string {
+    private get localId(): string {
         return this.context.connection.connectionId ?? "local";
     }
 
@@ -196,9 +197,8 @@ export class SpectatorCamera extends Behaviour {
      */
     private isSupportedPlatform() {
         const ua = window.navigator.userAgent;
-        const standalone = /Windows|MacOS/.test(ua);
         const isHololens = /Windows NT/.test(ua) && /Edg/.test(ua) && !/Win64/.test(ua);
-        return standalone && !isHololens;
+        return DeviceUtilities.isDesktop() && !DeviceUtilities.isMobileDevice() && !isHololens;
     }
 
     /**
@@ -268,11 +268,16 @@ export class SpectatorCamera extends Behaviour {
         const previousRenderTarget = renderer.getRenderTarget();
         let oldFramebuffer: WebGLFramebuffer | null = null;
 
+
         const webglState = renderer.state as WebGLState & { bindXRFramebuffer?: Function };
 
+
         // seems that in some cases, renderer.getRenderTarget returns null
         // even when we're rendering to a headset.
-        if (!previousRenderTarget) {
+        if (!previousRenderTarget || 
+            // Prevent rendering if in XR - @TODO: check if we need to allow this for VR?
+            previousRenderTarget["isXRRenderTarget"] === true) 
+        {
             if (!renderer.state.bindFramebuffer || !webglState.bindXRFramebuffer)
                 return;
 
@@ -557,12 +562,18 @@ class SpectatorSelectionController {
             for (const hit of hits) {
                 if (hit.distance < .2) continue;
                 const obj = hit.object;
-                const avatar = GameObject.getComponentInParent(obj, AvatarMarker);
-                const id = avatar?.connectionId;
+                // For WebXR
+                const state = PlayerState.getFor(obj);
+                let id = state?.owner;
+                // for SpectatorCamera
+                if (!id) {
+                    const avatar = GameObject.getComponentInParent(obj, AvatarMarker);
+                    id = avatar?.connectionId;
+                }
                 if (id) {
                     const view = this.context.players.getPlayerView(id);
                     this.spectator.target = view;
-                    if (debug) console.log("spectate", id, avatar);
+                    if (debug) console.log("spectate", id, state);
                     break;
                 }
             }
@@ -633,7 +644,7 @@ class SpectatorCamNetworking {
         this.context.connection.beginListen("spectator-request-follow", this._requestFollowMethod);
         this.context.connection.beginListen(RoomEvents.JoinedRoom, this._joinedRoomMethod);
         this.context.domElement.addEventListener("keydown", evt => {
-            if(!this.spectator.useKeys) return;
+            if (!this.spectator.useKeys) return;
             if (evt.key === "f") {
                 this.onRequestFollowMe();
             }
@@ -767,7 +778,7 @@ class SpectatorCamNetworking {
     }
 
     private _enforceFollowInterval: any;
-    
+
     /**
      * Periodically retries following a user if the initial attempt failed
      */

package/src/engine-components/api.ts

@@ -35,8 +35,8 @@
  */
 
 export * from "./codegen/components.js";
-export { Behaviour, Component, GameObject } from "./Component.js";
 export { Collider } from "./Collider.js"; // export abstract type
+export { Behaviour, Component, GameObject } from "./Component.js";
 
 // We dont want to export everything in the extensions
 export { ClearFlags } from "./Camera.js"

package/src/engine-components/webxr/Avatar.ts

@@ -2,6 +2,7 @@ import { Mesh, Object3D, Quaternion, Vector3 } from "three";
 
 import { AssetReference } from "../../engine/engine_addressables.js";
 import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js";
+import { ViewDevice } from "../../engine/engine_playerview.js";
 import { serializable } from "../../engine/engine_serialization_decorator.js";
 import type { IGameObject } from "../../engine/engine_types.js";
 import { getParam, PromiseAllWithErrors } from "../../engine/engine_utils.js";
@@ -55,10 +56,13 @@ export class Avatar extends Behaviour {
             const marker = this.gameObject.addComponent(AvatarMarker)!;
             marker.avatar = this.gameObject;
             marker.connectionId = playerstate.owner;
+
+            this.context.players.setPlayerView(playerstate.owner, this.head?.asset, ViewDevice.Headset);
         }
         else if (this.context.connection.isConnected) console.error("No player state found for avatar", this);
         // don't destroy the avatar when entering XR and not connected to a networking backend
         else if (playerstate && !this.context.connection.isConnected) playerstate.dontDestroy = true;
+
     }
 
     onLeaveXR(_args: NeedleXREventArgs): void {

package/src/engine-components/webxr/WebXR.ts

@@ -210,17 +210,25 @@ export class WebXR extends Behaviour {
             showBalloonWarning("<a href=\"https://developer.mozilla.org/en-US/docs/Web/API/WebXR_Device_API\" target=\"_blank\">WebXR</a> only works on secure connections (https).");
         }
 
-        if (this.useQuicklookExport) {
-            const existingUSDZExporter = GameObject.findObjectOfType(USDZExporter);
-            if (!existingUSDZExporter) {
-                // if no USDZ Exporter is found we add one and assign the scene to be exported
-                if (debug) console.log("WebXR: Adding USDZExporter");
-                this._usdzExporter = GameObject.addComponent(this.gameObject, USDZExporter);
-                this._usdzExporter.objectToExport = this.context.scene;
-                this._usdzExporter.autoExportAnimations = true;
-                this._usdzExporter.autoExportAudioSources = true;
+        // Showing the QuickLook button depends on whether we're on iOS or visionOS –
+        // on iOS we have AppClip support, so we don't need QuickLook button there,
+        // while on visionOS we use QuickLook for AR experiences for now (unless it happens to have WebXR support by then).
+        navigator.xr?.isSessionSupported("immersive-ar").catch(() => false).then((arSupported) => {
+
+            const isVisionOSFallback = DeviceUtilities.isVisionOS() && !arSupported;
+            
+            if (this.useQuicklookExport || isVisionOSFallback) {
+                const existingUSDZExporter = GameObject.findObjectOfType(USDZExporter);
+                if (!existingUSDZExporter) {
+                    // if no USDZ Exporter is found we add one and assign the scene to be exported
+                    if (debug) console.log("WebXR: Adding USDZExporter");
+                    this._usdzExporter = GameObject.addComponent(this.gameObject, USDZExporter);
+                    this._usdzExporter.objectToExport = this.context.scene;
+                    this._usdzExporter.autoExportAnimations = true;
+                    this._usdzExporter.autoExportAudioSources = true;
+                }
             }
-        }
+        });
 
         this.handleCreatingHTML();
         this.handleOfferSession();
@@ -360,7 +368,7 @@ export class WebXR extends Behaviour {
 
         // Handle AR session root
         if (args.xr.isAR) {
-            let sessionroot = GameObject.findObjectOfType(WebARSessionRoot);
+            let sessionroot = GameObject.findObjectOfType(WebARSessionRoot, this.context, false);
             // Only create a WebARSessionRoot if none is in the scene already
             if (!sessionroot) {
                 if (this.usePlacementReticle) {