@needle-tools/engine 4.12.0-next.ca2cebd → 4.12.0-next.da19dd0

package/src/engine/xr/NeedleXRSession.ts

@@ -6,6 +6,7 @@ import { Context, FrameEvent } from "../engine_context.js";
 import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
 import { isDestroyed } from "../engine_gameobject.js";
 import { Gizmos } from "../engine_gizmos.js";
+import { Telemetry } from "../engine_license.js";
 import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
 import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
 import type { ICamera, IComponent, INeedleXRSession } from "../engine_types.js";
@@ -82,6 +83,34 @@ function getDOMOverlayElement(domElement: HTMLElement) {
 handleSessionGranted();
 async function handleSessionGranted() {
 
+    // await delay(400);
+
+
+    let defaultMode: XRSessionMode = "immersive-vr";
+
+    try {
+        // In app clips we default to AR
+        if (DeviceUtilities.isNeedleAppClip()) {
+            defaultMode = "immersive-ar";
+        }
+        // Check if VR is even supported, otherwise try AR
+        else if (!(await navigator.xr?.isSessionSupported("immersive-vr"))) {
+            defaultMode = "immersive-ar";
+        }
+        // Check if AR is supported, otherwise we can't do anything
+        if (!(await navigator.xr?.isSessionSupported("immersive-ar")) && defaultMode === "immersive-ar") {
+            // console.warn("[NeedleXRSession:granted] Neither VR nor AR supported, aborting session start.");
+            // showBalloonMessage("NeidleXRSession: Neither VR nor AR supported, aborting session start.");
+            return;
+        }
+    } catch (e) {
+        console.debug("[NeedleXRSession:granted] Error while checking XR support:", e);
+        // showBalloonWarning("NeedleXRSession: Error while checking XR support: " + (e as Error).message);
+        return;
+    }
+
+    // showBalloonMessage("sessiongranted: " + defaultMode);
+
     // TODO: asap session granted doesnt handle the pre-room yet
     if (getParam("debugasap")) {
         let asapSession = globalThis["needle:XRSession"] as XRSession | undefined | Promise<XRSession>;
@@ -94,11 +123,11 @@ async function handleSessionGranted() {
                 enableSpatialConsole(true);
                 const session = await asapSession;
                 if (session) {
-                    const sessionInit = NeedleXRSession.getDefaultSessionInit("immersive-vr");
-                    NeedleXRSession.setSession("immersive-vr", session, sessionInit, cb.context);
+                    const sessionInit = NeedleXRSession.getDefaultSessionInit(defaultMode);
+                    NeedleXRSession.setSession(defaultMode, session, sessionInit, cb.context);
                 }
                 else {
-                    console.error("NeedleXRSession: ASAP session was rejected");
+                    console.error("[NeedleXRSession:granted] ASAP session was rejected");
                 }
                 asapSession = undefined;
             });
@@ -106,6 +135,9 @@ async function handleSessionGranted() {
         }
     }
 
+    // console.log("Attaching sessiongranted handler...", { haveXR: 'xr' in navigator });
+    // setTimeout(() => console.log("Session Granted handler attached.", { haveXR: 'xr' in navigator }), 1000);
+
     if ('xr' in navigator) {
         // WebXRViewer (based on Firefox) has a bug where addEventListener
         // throws a silent exception and aborts execution entirely.
@@ -115,10 +147,8 @@ async function handleSessionGranted() {
         }
 
         navigator.xr?.addEventListener('sessiongranted', async () => {
-            enableSpatialConsole(true);
+            // enableSpatialConsole(true);
 
-            console.log("Received Session Granted...")
-            await delay(100);
 
             const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode") as XRSessionMode;
             const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null;
@@ -126,7 +156,8 @@ async function handleSessionGranted() {
 
             let info: SessionInfo | null = null;
             if (contextIsLoading()) {
-                await TemporaryXRContext.start(lastSessionMode || "immersive-vr", init || NeedleXRSession.getDefaultSessionInit("immersive-vr"));
+                await TemporaryXRContext.start(lastSessionMode || defaultMode, init || NeedleXRSession.getDefaultSessionInit(defaultMode))
+                    .catch(e => console.warn("[NeedleXRSession:granted] TemporaryXRContext start failed:", e));
                 await waitForContextLoadingFinished();
                 info = await TemporaryXRContext.handoff();
             }
@@ -134,17 +165,19 @@ async function handleSessionGranted() {
                 NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
             }
             else if (lastSessionMode && lastSessionInit) {
-                console.log("Session Granted: Restore last session")
+                console.log("[NeedleXRSession:granted] Restore last session")
                 const init = JSON.parse(lastSessionInit);
                 NeedleXRSession.start(lastSessionMode as XRSessionMode, init).catch(e => console.warn(e));
             }
             else {
                 // if no session was found we start VR by default
-                NeedleXRSession.start("immersive-vr").catch(e => console.warn("Session Granted failed:", e));
+                NeedleXRSession.start(defaultMode).catch(e => console.warn("[NeedleXRSession:granted] failed:", e));
             }
             // make sure we only subscribe to the event once
         }, { once: true });
-
+    }
+    else {
+        // showBalloonWarning("NeedleXRSession: WebXR not available in this browser.");
     }
 }
 function saveSessionInfo(mode: XRSessionMode, init: XRSessionInit) {
@@ -419,27 +452,78 @@ export class NeedleXRSession implements INeedleXRSession {
      * @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit
      * @param context The Needle Engine context to use
      */
-    static async start(mode: XRSessionMode | "ar", init?: XRSessionInit, context?: Context): Promise<NeedleXRSession | null> {
+    static async start(mode: XRSessionMode | "ar" | "quicklook", init?: XRSessionInit, context?: Context): Promise<NeedleXRSession | null> {
 
-        // handle iOS platform where "immersive-ar" is not supported
+        // handle iOS platform where "immersive-ar" is special:
+        // - we either launch QuickLook
+        // - or forward to the Needle App Clip experience for WebXR AR
         // TODO: should we add a separate mode (e.g. "AR")? https://linear.app/needle/issue/NE-5303
         if (DeviceUtilities.isiOS()) {
-            if (mode === "ar") {
-                const arSupported = await this.isARSupported();
-                if (!arSupported) {
-                    InternalUSDZRegistry.exportAndOpen();
-                    return null;
+
+            const arSupported = await this.isARSupported().catch(() => false);
+
+            // On VisionOS, we use QuickLook for AR experiences; no AppClip support for now.
+            if (DeviceUtilities.isVisionOS() && !arSupported && (mode === "ar" || mode === "immersive-ar")) {
+                mode = "quicklook";
+            }
+
+            if (mode === "quicklook") {
+                Telemetry.sendEvent(Context.Current, "xr", {
+                    action: "quicklook_export",
+                    source: "NeedleXRSession.start",
+                });
+                InternalUSDZRegistry.exportAndOpen();
+                return null;
+            }
+
+            if (!arSupported && (mode === "immersive-ar" || mode === "ar")) {
+                // const debugAppClip = getParam("debugappclip")
+                // Forward to the AppClip experience (Using the apple.com url the appclip overlay shows immediately)
+                // const url =`https://appclip.needle.tools/ar?url=${(location.href)}`;
+                const url = new URL("https://appclip.apple.com/id?p=tools.needle.launch-app.Clip");
+                url.searchParams.set("url", location.href);
+
+                const urlStr = url.toString();
+
+                Telemetry.sendEvent(Context.Current, "xr", {
+                    action: "app_clip_launch",
+                    source: "NeedleXRSession.start",
+                    url: urlStr,
+                });
+
+                // if we are in an iframe, we need to navigate the top window
+                const topWindow = window.top || window;
+                try {
+                    console.debug("iOS device detected - opening Needle App Clip for AR experience", { mode, init, url });
+                    // navigate to app clip url but keep the current url in history, open in same tab
+                    // eslint-disable-next-line xss/no-location-href-assign
+                    topWindow.location.href = urlStr;
                 }
-                else {
-                    mode = "immersive-ar";
+                catch (e) {
+                    console.warn("Error navigating to AppClip " + urlStr + "\n", e);
+                    // if top window navigation fails and we are in an iframe, we try to navigate the top window directly
+                    const weAreInIframe = window !== window.top;
+                    if (weAreInIframe) {
+                        // we can try to open a new tab as a fallback
+                        window.open(urlStr, "_blank");
+                    }
+                    // eslint-disable-next-line xss/no-location-href-assign
+                    else window.location.href = urlStr;
                 }
+
+                return null;
             }
         }
-        else if (mode == "ar") {
-            mode = "immersive-ar";
-        }
 
+        if (mode === "quicklook") {
+            console.warn("QuickLook mode is only supported on iOS devices");
+            return null;
+        }
 
+        // Since we now know we are not on iOS, ar mode becomes "immersive-ar"
+        if (mode == "ar") {
+            mode = "immersive-ar";
+        }
 
         if (isDevEnvironment() && getParam("debugxrpreroom")) {
             console.warn("Debug: Starting temporary XR session");
@@ -535,12 +619,18 @@ export class NeedleXRSession implements INeedleXRSession {
             listener({ mode, init });
         }
         if (debug) showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")");
+        Telemetry.sendEvent(Context.Current, "xr", {
+            action: "session_request",
+            mode: mode,
+            features: ((init.requiredFeatures ?? []).concat(init.optionalFeatures ?? [])).join(","),
+            source: "NeedleXRSession.start",
+        });
         this._currentSessionRequest = navigator?.xr?.requestSession(mode, init);
         this._currentSessionRequestMode = mode;
         /**@type {XRSystem} */
         const newSession = await (this._currentSessionRequest)?.catch(e => {
-            console.error(e, "Code: " + e.code);
-            if (e.code === 9) showBalloonWarning("Make sure your device has the required permissions (e.g. camera access)")
+            console.error(e, "Code: " + e?.code);
+            if (e?.code === 9) showBalloonWarning("Couldn't start XR session. Make sure you allow the required permissions.")
             console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)");
             const notSecure = location.protocol === 'http:';
             if (notSecure) showBalloonWarning("XR requires a secure connection (HTTPS)");
@@ -1111,6 +1201,12 @@ export class NeedleXRSession implements INeedleXRSession {
 
         console.debug("XR Session ended");
 
+        Telemetry.sendEvent(Context.Current, "xr", {
+            action: "session_end",
+            mode: this.mode,
+            source: "NeedleXRSession.onEnd",
+        });
+
         deleteSessionInfo();
 
         this.onAfterRender();

package/src/engine/xr/TempXRContext.ts

@@ -32,7 +32,13 @@ export class TemporaryXRContext {
                 return null;
             }
             this._requestInFlight = true;
-            const session = await navigator.xr.requestSession(mode, init);
+            const session = await navigator.xr.requestSession(mode, init).catch(err => {
+                console.error("Failed to start temporary XR session:", err);
+            });
+            if (!session) {
+                this._requestInFlight = false;
+                return null;
+            }
             session.addEventListener("end", () => {
                 this._active = null;
             });
@@ -82,10 +88,14 @@ export class TemporaryXRContext {
         this._session = session;
         this._session.addEventListener("end", this.onEnd);
 
-        this._renderer = new WebGLRenderer({ alpha: true });
+        this._renderer = new WebGLRenderer({ alpha: true, antialias: true }); 
+        this._renderer.outputColorSpace = 'srgb';
         this._renderer.setAnimationLoop(this.onFrame);
         this._renderer.xr.setSession(session);
         this._renderer.xr.enabled = true;
+        // Set pixel ratio and size
+        this._renderer.setPixelRatio(Math.min(2, window.devicePixelRatio));
+        this._renderer.setSize(window.innerWidth, window.innerHeight);
         this._camera = new PerspectiveCamera();
         this._scene = new Scene();
         this._scene.fog = new Fog(0x444444, 10, 250);

package/src/engine/xr/usdz.ts

@@ -1,3 +1,4 @@
+import { isDevEnvironment } from "../debug/index.js";
 
 declare type USDZExporter = {
     exportAndOpen(): Promise<any>,
@@ -11,7 +12,11 @@ export namespace InternalUSDZRegistry {
     const usdzExporter: USDZExporter[] = [];
 
     export function exportAndOpen(): boolean {
-        if (!usdzExporter?.length) return false;
+        if (!usdzExporter?.length) {
+            if (isDevEnvironment()) {
+                console.warn("No USDZ exporters found – cannot export USDZ for QuickLook.");
+            }
+        }
         for (const exp of usdzExporter) {
             exp.exportAndOpen();
         }

package/src/engine-components/AlignmentConstraint.ts

@@ -9,7 +9,7 @@ import { Behaviour, GameObject } from "./Component.js";
  * You can use this to create dynamic beams or connectors between objects.
  *
  * @summary Aligns and scales the object between two target GameObjects
- * @category Utility
+ * @category Constraints
  * @group Components 
  **/
 export class AlignmentConstraint extends Behaviour {

package/src/engine-components/Animation.ts

@@ -60,7 +60,7 @@ class Vec2 { x!: number; y!: number }
 /**
  * Animation component to play animations on a GameObject.
  * 
- * @summary Play animations
+ * @summary Plays animations from AnimationClips 
  * @category Animation and Sequencing
  * @group Components
  */

package/src/engine-components/Animator.ts

@@ -43,7 +43,7 @@ export declare class PlayOptions {
  * It works with an {@link AnimatorController} to handle state transitions and animation blending.
  * A new AnimatorController can be created from code via `AnimatorController.createFromClips`.  
  * 
- * @summary Animation controller and playback component
+ * @summary Plays and manages animations on a GameObject based on an AnimatorController
  * @category Animation and Sequencing
  * @group Components
 */

package/src/engine-components/AudioListener.ts

@@ -9,7 +9,7 @@ import { Behaviour, GameObject } from "./Component.js";
  * This component creates and manages a Three.js {@link three#AudioListener}, automatically connecting it
  * to the main camera or a Camera in the parent hierarchy.
  * 
- * @summary Audio listener for 3D audio capture
+ * @summary Receives audio in the scene and outputs it to speakers
  * @category Multimedia
  * @group Components
  */

package/src/engine-components/AudioSource.ts

@@ -52,7 +52,7 @@ export enum AudioRolloffMode {
  * is muted, the volume is set to 0. When unmuted, the volume
  * returns to its previous value.
  * 
- * @summary 3D audio source with spatial positioning and playback controls
+ * @summary Plays audio clips from files or media streams
  * @category Multimedia
  * @group Components
  */

package/src/engine-components/Camera.ts

@@ -8,7 +8,7 @@ import { Context } from "../engine/engine_setup.js";
 import { RenderTexture } from "../engine/engine_texture.js";
 import { getTempColor, getWorldPosition } from "../engine/engine_three_utils.js";
 import type { ICamera } from "../engine/engine_types.js"
-import { getParam } from "../engine/engine_utils.js";
+import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
 import { NeedleXREventArgs } from "../engine/engine_xr.js";
 import { RGBAColor } from "../engine/js-extensions/index.js";
 import { Behaviour, GameObject } from "./Component.js";
@@ -37,7 +37,7 @@ const debugscreenpointtoray = getParam("debugscreenpointtoray");
  * Internally, this component uses {@link PerspectiveCamera} and {@link OrthographicCamera} three.js objects.
  * 
  * @summary Rendering scenes from a specific viewpoint
- * @category Camera
+ * @category Camera and Controls
  * @group Components
  */
 export class Camera extends Behaviour implements ICamera {
@@ -664,6 +664,9 @@ export class Camera extends Behaviour implements ICamera {
                 else if (navigator.userAgent?.includes("Mozilla") && navigator.userAgent?.includes("Mobile WebXRViewer/v2")) {
                     transparent = true;
                 }
+                else if(DeviceUtilities.isNeedleAppClip()) {
+                    return true;
+                }
             }
         }
 

package/src/engine-components/CharacterController.ts

@@ -110,8 +110,12 @@ export class CharacterController extends Behaviour {
 }
 
 /**
- * @category Camera
- * @category Interactivity
+ * CharacterControllerInput handles user input to control a CharacterController.  
+ * It supports movement, looking around, jumping, and double jumping.  
+ * You can customize movement speed, rotation speed, and jump forces.  
+ * It also integrates with an Animator component for character animations.
+ * @summary User Input for Character Controller
+ * @category Character
  * @group Components
  */
 export class CharacterControllerInput extends Behaviour {

package/src/engine-components/Collider.ts

@@ -26,7 +26,7 @@ import { Rigidbody } from "./RigidBody.js";
  * @category Physics  
  * @group Components
  */
-export class Collider extends Behaviour implements ICollider {
+export abstract class Collider extends Behaviour implements ICollider {
 
     /**
      * Identifies this component as a collider.

package/src/engine-components/Component.ts

@@ -476,10 +476,11 @@ export abstract class GameObject extends Object3D implements Object3D, IGameObje
      * Gets a component of the specified type in the gameObject's children hierarchy
      * @param go GameObject to search in
      * @param typeName Constructor of the component type
+     * @param includeInactive Whether to include inactive objects in the search
      * @returns The first matching component if found, otherwise null
      */
-    public static getComponentInChildren<T extends IComponent>(go: IGameObject | Object3D, typeName: Constructor<T>): T | null {
-        return getComponentInChildren(go, typeName);
+    public static getComponentInChildren<T extends IComponent>(go: IGameObject | Object3D, typeName: Constructor<T>, includeInactive: boolean = false): T | null {
+        return getComponentInChildren(go, typeName, includeInactive);
     }
 
     /**
@@ -588,7 +589,7 @@ export abstract class Component implements IComponent, EventTarget,
      */
     get [$componentName]() { return TypeStore.getKey(this.constructor as any) || undefined; }
 
-    
+
     private __context: Context | undefined;
 
     /**
@@ -1106,7 +1107,7 @@ export abstract class Component implements IComponent, EventTarget,
             this.dispatchEvent(new CustomEvent("destroyed", { detail: this }));
         }
         destroyComponentInstance(this as any);
-        if(isHotReloadEnabled()) unregisterHotReloadType(this);
+        if (isHotReloadEnabled()) unregisterHotReloadType(this);
     }
 
     /**

package/src/engine-components/DragControls.ts

@@ -447,7 +447,7 @@ export class DragControls extends Behaviour implements IPointerEventHandler {
         if (!this || !this._isDragging) return;
         this._isDragging = false;
         for (const rb of this._draggingRigidbodies) {
-            rb.setVelocity(rb.smoothedVelocity);
+            rb.setVelocity(rb.smoothedVelocity.multiplyScalar(this.context.time.deltaTime));
         }
         this._draggingRigidbodies.length = 0;
         this._targetObject = null;
@@ -485,6 +485,8 @@ interface IDragHandler {
     onDragUpdate?(numberOfPointers: number): void;
 }
 
+
+// #region MultiTouchDragHandler
 /** 
  * Handles two touch points affecting one object. 
  * Enables multi-touch interactions that allow movement, scaling, and rotation of objects.
@@ -734,6 +736,7 @@ class MultiTouchDragHandler implements IDragHandler {
 }
 
 
+// #region DragPointerHandler
 /** 
  * Handles a single pointer on an object. 
  * DragPointerHandlers manage determining if a drag operation has started, tracking pointer movement,
@@ -1335,6 +1338,7 @@ class DragPointerHandler implements IDragHandler {
     }
 }
 
+// #region LegacyDragVisualsHelper
 /** 
  * Provides visual helper elements for DragControls.
  * Shows where objects will be placed and their relation to surfaces below them.

package/src/engine-components/Joints.ts

@@ -36,6 +36,12 @@ export abstract class Joint extends Behaviour {
     protected abstract createJoint(self: Rigidbody, other: Rigidbody);
 }
 
+/**
+ * The FixedJoint groups together 2 rigidbodies, making them stick together in their bound position
+ * @summary Connect two Rigidbodies and make them stick together
+ * @category Physics
+ * @group Components
+ */
 export class FixedJoint extends Joint {
 
     protected createJoint(self: Rigidbody, other: Rigidbody) {
@@ -43,6 +49,14 @@ export class FixedJoint extends Joint {
     }
 }
 
+/**
+ * The HingeJoint groups together 2 rigid bodies, constraining them to move like connected by a hinge.
+ * 
+ * You can specify the anchor point and axis of rotation for the hinge.
+ * @summary Connect two Rigidbodies with a hinge
+ * @category Physics
+ * @group Components
+ */
 export class HingeJoint extends Joint {
 
     @serializable(Vector3)

package/src/engine-components/LookAtConstraint.ts

@@ -9,7 +9,7 @@ import type { OrbitControls } from "./OrbitControls.js";
  * This component is used by {@link OrbitControls} internally.
  * 
  * @summary Look At Constraint for OrbitControls
- * @category Camera Controls
+ * @category Camera and Controls
  * @group Components
  */
 export class LookAtConstraint extends Behaviour {

package/src/engine-components/OrbitControls.ts

@@ -64,7 +64,7 @@ declare module 'three/examples/jsm/controls/OrbitControls.js' {
  * The three OrbitControls object can be accessed via the `controls` property.  
  * The object being controlled by the OrbitControls (usually the camera) can be accessed via the `controllerObject` property.
  * @summary Camera controller using three.js OrbitControls
- * @category Camera
+ * @category Camera and Controls
  * @group Components
  */
 export class OrbitControls extends Behaviour implements ICameraController {

package/src/engine-components/Renderer.ts

@@ -805,6 +805,12 @@ export class Renderer extends Behaviour implements IRenderer {
 export class MeshRenderer extends Renderer {
 }
 
+/**
+ * Renders deformable meshes that deform via bones and/or blend shapes.
+ * @summary Renderer for deformable meshes
+ * @category Rendering
+ * @group Components
+ **/
 export class SkinnedMeshRenderer extends MeshRenderer {
 
     private _needUpdateBoundingSphere = false;

package/src/engine-components/RendererInstancing.ts

@@ -669,14 +669,17 @@ class InstancedMeshRenderer {
     private grow(geometry: BufferGeometry) {
         const id = ++this._growId;
         const growFactor = 2;
-        const newSize = Math.ceil(this._maxInstanceCount * growFactor);
+
+        const growInstances = this.count >= this._maxInstanceCount;
+        const newSize = growInstances ? Math.ceil(this._maxInstanceCount * growFactor) : this._maxInstanceCount;
 
         // create a new BatchedMesh instance
         // TODO: we should keep track of how many instances for each geometry we have and consider that when estimating new space
         const estimatedSpace = this.tryEstimateVertexCountSize(newSize, [geometry]);// geometry.attributes.position.count;
         // const indices = geometry.index ? geometry.index.count : 0;
-        const newMaxVertexCount = Math.max(this._maxVertexCount, estimatedSpace.vertexCount);
-        const newMaxIndexCount = Math.max(this._maxIndexCount, estimatedSpace.indexCount);//, Math.ceil(this._maxVertexCount * growFactor));
+        const vertexGrowFactor = 1.25;
+        const newMaxVertexCount = Math.max(this._maxVertexCount, Math.ceil(estimatedSpace.vertexCount * vertexGrowFactor));
+        const newMaxIndexCount = Math.max(this._maxIndexCount, Math.ceil(estimatedSpace.indexCount * vertexGrowFactor));
 
         if (debugInstancing) {
             const geometryInfo = getMeshInformation(geometry);