@needle-tools/engine 4.12.0-next.c6c0281 → 4.12.0-next.da19dd0

package/src/engine/engine_license.ts

@@ -1,9 +1,13 @@
+import { dof } from "three/src/nodes/TSL.js";
 import { isDevEnvironment } from "./debug/index.js";
 import { BUILD_TIME, GENERATOR, PUBLIC_KEY, VERSION } from "./engine_constants.js";
 import { ContextEvent, ContextRegistry } from "./engine_context_registry.js";
+import { onInitialized } from "./engine_lifecycle_api.js";
+import { isLocalNetwork } from "./engine_networking_utils.js";
 import { Context } from "./engine_setup.js";
 import type { IContext } from "./engine_types.js";
 import { getParam } from "./engine_utils.js";
+import { InternalAttributeUtils } from "./engine_utils_attributes.js";
 
 const debug = getParam("debuglicense");
 
@@ -67,6 +71,169 @@ function invokeLicenseCheckResultChanged(result: boolean) {
     }
 }
 
+
+
+// #region Telemetry
+export namespace Telemetry {
+
+    window.addEventListener("error", (event: ErrorEvent) => {
+        sendError(Context.Current, "unhandled_error", event);
+    });
+    window.addEventListener("unhandledrejection", (event: PromiseRejectionEvent) => {
+        sendError(Context.Current, "unhandled_promise_rejection", {
+            message: event.reason?.message,
+            stack: event.reason?.stack,
+            timestamp: Date.now(),
+        });
+    });
+
+    onInitialized((ctx => sendPageViewEvent(ctx)), { once: true });
+
+    function sendPageViewEvent(ctx: IContext) {
+        if (!isAllowed(ctx)) {
+            if (debug) console.debug("Telemetry is disabled via no-telemetry attribute");
+            return;
+        }
+        return doFetch({
+            site_id: "dabb8317376f",
+            type: "pageview",
+            pathname: window.location.pathname,
+            hostname: window.location.hostname,
+            page_title: document.title,
+            referrer: document.referrer,
+            user_agent: navigator.userAgent,
+            querystring: window.location.search,
+            language: navigator.language,
+            screenWidth: window.screen.width,
+            screenHeight: window.screen.height,
+            event_name: "page_view"
+        }).then(res => {
+            if (res instanceof Response && res.ok && isLocalNetwork()) {
+                sendEvent(ctx, "info", {
+                    src: ctx.domElement?.getAttribute("src") || "",
+                    version: VERSION,
+                    generator: GENERATOR,
+                    build_time: BUILD_TIME,
+                    public_key: PUBLIC_KEY,
+                });
+            }
+        })
+    }
+
+    export function isAllowed(context: IContext | null | undefined): boolean {
+        let domElement = context?.domElement as HTMLElement | null;
+        if (!domElement) domElement = document.querySelector<HTMLElement>("needle-engine");
+        if (!domElement && !context) return false;
+
+        const attribute = domElement?.getAttribute("no-telemetry");
+        if (attribute === "" || attribute === "true" || attribute === "1") {
+            if (NEEDLE_ENGINE_LICENSE_TYPE === "pro" || NEEDLE_ENGINE_LICENSE_TYPE === "enterprise") {
+                if (debug) console.debug("Telemetry is disabled via no-telemetry attribute");
+                return false;
+            }
+        }
+        return true;
+    }
+
+    const id = "dabb8317376f";
+
+    /**
+     * Sends a telemetry event
+     */
+    export async function sendEvent(context: IContext | null | undefined, eventName: string, properties?: Record<string, any>) {
+        if (!isAllowed(context)) {
+            if (debug) console.debug("Telemetry is disabled");
+            return;
+        }
+        const body = {
+            site_id: id,
+            type: "custom_event",
+            pathname: window.location.pathname,
+            event_name: eventName,
+            properties: properties ? JSON.stringify(properties) : undefined,
+        }
+        return doFetch(body);
+    }
+
+    type ErrorData = {
+        message?: string;
+        stack?: string;
+        filename?: string;
+        lineno?: number;
+        colno?: number;
+        timestamp?: number;
+    }
+
+    export async function sendError(context: IContext, errorName: string, error: ErrorData | ErrorEvent | Error) {
+
+        if (!isAllowed(context)) {
+            if (debug) console.debug("Telemetry is disabled");
+            return;
+        }
+
+        if (error instanceof ErrorEvent) {
+            error = {
+                message: error.message,
+                stack: error.error?.stack,
+                filename: error.filename,
+                lineno: error.lineno,
+                colno: error.colno,
+                timestamp: error.timeStamp || Date.now(),
+
+            };
+        }
+        else if (error instanceof Error) {
+            error = {
+                message: error.message,
+                stack: error.stack,
+                timestamp: Date.now(),
+            };
+        }
+        const body = {
+            site_id: id,
+            type: "error",
+            event_name: errorName || "error",
+            properties: JSON.stringify({
+                error_name: errorName,
+                message: error.message,
+                stack: error.stack,
+                filename: error.filename,
+                lineno: error.lineno,
+                colno: error.colno,
+                timestamp: error.timestamp,
+            })
+        }
+        return doFetch(body);
+    }
+
+    function doFetch(body: Record<string, any>) {
+        try {
+            const url = "https://needle.tools/api/v1/rum/t";
+            return fetch(url, {
+                method: "POST",
+                body: JSON.stringify(body),
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                // Ensures request completes even if page unloads
+                keepalive: true,
+                // Allow CORS requests
+                mode: 'cors',
+                // Low priority to avoid blocking other requests
+                // @ts-ignore
+                priority: 'low',
+            }).catch(e => {
+                if (debug) console.error("Failed to send telemetry", e);
+            })
+        }
+        catch (err) {
+            if (debug) console.error(err);
+        }
+        return Promise.resolve();
+    }
+}
+
+
 ContextRegistry.registerCallback(ContextEvent.ContextRegistered, evt => {
     showLicenseInfo(evt.context);
     handleForbidden(evt.context);
@@ -341,14 +508,9 @@ async function sendUsageMessageToAnalyticsBackend(context: IContext) {
     // We can't send beacons from cross-origin isolated pages
     if (window.crossOriginIsolated) return;
 
-    const licenseType = NEEDLE_ENGINE_LICENSE_TYPE;
-    if (licenseType === "pro" || licenseType === "enterprise") {
-        const attribute = context?.domElement?.getAttribute("no-telemetry");
-        if (attribute === "" || attribute === "true" || attribute === "1") {
-            if (debug) console.debug("Telemetry is disabled");
-            return;
-        }
-        if (debug) console.debug("Telemetry attribute: " + attribute);
+    if (!Telemetry.isAllowed(context)) {
+        if (debug) console.debug("Telemetry is disabled via no-telemetry attribute");
+        return;
     }
 
     try {
@@ -379,4 +541,4 @@ async function sendUsageMessageToAnalyticsBackend(context: IContext) {
         if (debug)
             console.log("Failed to send non-commercial usage message to analytics backend", err);
     }
-}
+}

package/src/engine/engine_networking.ts

@@ -629,7 +629,7 @@ export class NetworkConnection implements INetworkConnection {
                 return;
             }
 
-            console.debug("⊡ Connecting to networking backend on\n" + networkingServerUrl)
+            console.debug("Connecting to networking backend on\n" + networkingServerUrl)
             const pkg = await import('websocket-ts');
             // @ts-ignore
             const WebsocketBuilder = pkg.default?.WebsocketBuilder ?? pkg.WebsocketBuilder;
@@ -641,8 +641,8 @@ export class NetworkConnection implements INetworkConnection {
                     this._connectingToWebsocketPromise = null;
                     this._ws = ws;
                     this.connected = true;
-                    if (isDevEnvironment() || debugNet) console.log("⊞ Connected to networking backend\n" + networkingServerUrl);
-                    else console.debug("⊞ Connected to networking backend", networkingServerUrl);
+                    if (isDevEnvironment() || debugNet) console.log("Connected to networking backend\n" + networkingServerUrl);
+                    else console.debug("Connected to networking backend", networkingServerUrl);
                     resolve(true);
                     this.onSendQueued(SendQueue.OnConnection);
                 })
@@ -656,10 +656,10 @@ export class NetworkConnection implements INetworkConnection {
                     console.error(msg);
                 })
                 .onError((_e) => {
-                    console.error("⊠ Websocket connection failed...");
+                    console.error("Websocket connection failed...");
                     resolve(false);
                 })
-                .onRetry(() => { console.log("→ Retry connecting to networking websocket") })
+                .onRetry(() => { console.log("Retry connecting to networking websocket") })
                 .build();
             ws.addEventListener(pkg.WebsocketEvent.message, (socket, msg) => {
                 this.onMessage(socket, msg);

package/src/engine/engine_physics.ts

@@ -1,7 +1,7 @@
 import { getRaycastMesh } from '@needle-tools/gltf-progressive';
 import { ArrayCamera, Box3, BufferGeometry, Camera, type Intersection, Layers, Line, Matrix3, Matrix4, Mesh, Object3D, PerspectiveCamera, Plane, Ray, Raycaster, SkinnedMesh, Sphere, SphereGeometry, Vector2, Vector3 } from 'three'
 import { GroundedSkybox } from 'three/examples/jsm/objects/GroundedSkybox.js';
-import type { MeshBVH, MeshBVHOptions, StaticGeometryGenerator } from 'three-mesh-bvh';
+import type { ComputeBVHOptions, GeometryBVH, MeshBVH, StaticGeometryGenerator } from 'three-mesh-bvh';
 import type { GenerateMeshBVHWorker } from 'three-mesh-bvh/src/workers/GenerateMeshBVHWorker.js';
 
 import { isDevEnvironment } from './debug/index.js';
@@ -762,7 +762,7 @@ namespace NEMeshBVH {
             return true;
         }
         else if (method instanceof Sphere) {
-            const bvh = geom.boundsTree;
+            const bvh = geom.boundsTree as MeshBVH | undefined;
             if (bvh) {
                 const sphere = method;
                 // Gizmos.DrawWireSphere(sphere.center, sphere.radius, 0xdddd00, 1, false);
@@ -789,7 +789,7 @@ namespace NEMeshBVH {
     let _acceleratedRaycast: Function | null = null;
     let _MeshBVH: ConstructorConcrete<MeshBVH> | null = null;
     let _StaticGeometryGenerator: ConstructorConcrete<StaticGeometryGenerator> | null = null;
-    let _computeBoundsTree: ((_opt?: MeshBVHOptions) => MeshBVH) | null = null;
+    let _computeBoundsTree: (( options?: ComputeBVHOptions ) => GeometryBVH) | null = null;
 
     function loadMeshBVHLibrary() {
         if (didLoadMeshBVHLibrary) return;

package/src/engine/engine_serialization_builtin_serializer.ts

@@ -337,7 +337,7 @@ class EventListSerializer extends TypeSerializer {
                             args = call.arguments.map(deserializeArgument);
                         }
                         const method = target[call.method];
-                        if (!method) {
+                        if (method === undefined) {
                             console.warn(`EventList method not found: \"${call.method}\" on ${target?.name}`);
                         }
                         else {

package/src/engine/engine_utils.ts

@@ -606,7 +606,8 @@ export namespace DeviceUtilities {
     /** @returns `true` if we're currently on an iPad */
     export function isiPad() {
         if (__isiPad !== undefined) return __isiPad;
-        return __isiPad = /iPad/.test(navigator.userAgent);
+        const userAgent = navigator.userAgent.toLowerCase();
+        return __isiPad = /iPad/.test(navigator.userAgent) || userAgent.includes("macintosh") && "ontouchend" in document;
     }
 
     let __isAndroidDevice: boolean | undefined;
@@ -623,17 +624,25 @@ export namespace DeviceUtilities {
         return __isMozillaXR = /WebXRViewer\//i.test(navigator.userAgent);
     }
 
+    let __isNeedleAppClip: boolean | undefined;
+    /** @returns `true` if we're currently in the Needle App Clip */
+    export function isNeedleAppClip() {
+        if (__isNeedleAppClip !== undefined) return __isNeedleAppClip;
+        return __isNeedleAppClip = /NeedleAppClip\//i.test(navigator.userAgent);
+    }
+
     let __isMacOS: boolean | undefined;
     // https://developer.mozilla.org/en-US/docs/Web/API/Navigator/userAgentData
     /** @returns `true` for MacOS devices */
     export function isMacOS() {
         if (__isMacOS !== undefined) return __isMacOS;
+        if (isiOS() || isiPad()) return __isMacOS = false;
+        const userAgent = navigator.userAgent.toLowerCase();
         if (navigator.userAgentData) {
             // Use modern UA Client Hints API if available
             return __isMacOS = navigator.userAgentData.platform === 'macOS';
         } else {
             // Fallback to user agent string parsing
-            const userAgent = navigator.userAgent.toLowerCase();
             return __isMacOS = userAgent.includes('mac os x') || userAgent.includes('macintosh');
         }
     }
@@ -642,13 +651,13 @@ export namespace DeviceUtilities {
     /** @returns `true` for VisionOS devices */
     export function isVisionOS() {
         if (__isVisionOS !== undefined) return __isVisionOS;
-        return __isVisionOS = (isMacOS() && "xr" in navigator);
+        return __isVisionOS = (isiPad() && "xr" in navigator && supportsQuickLookAR());
     }
 
     let __isiOS: boolean | undefined;
     const iosDevices = ['iPad Simulator', 'iPhone Simulator', 'iPod Simulator', 'iPad', 'iPhone', 'iPod'];
 
-    /** @returns `true` for iOS devices like iPad, iPhone, iPod... */
+    /** @returns `true` for mobile Apple devices like iPad, iPhone, iPod, Vision Pro, ... */
     export function isiOS() {
         if (__isiOS !== undefined) return __isiOS;
         // eslint-disable-next-line deprecation/deprecation
@@ -725,6 +734,16 @@ export namespace DeviceUtilities {
         else __chromeVersion = null;
         return __chromeVersion;
     }
+
+    let __safariVersion: string | null | undefined;
+    export function getSafariVersion() {
+        if (__safariVersion !== undefined) return __safariVersion;
+        const match = navigator.userAgent.match(/Version\/(\d+\.\d+)/);
+        if (match && isSafari()) {
+            __safariVersion = match[1];
+        } else __safariVersion = null;
+        return __safariVersion;
+    }
 }
 
 /**

package/src/engine/webcomponents/WebXRButtons.ts

@@ -57,13 +57,16 @@ export class WebXRButtonFactory {
         this._quicklookButton = button;
         button.dataset["needle"] = "quicklook-button";
         const supportsQuickLook = DeviceUtilities.supportsQuickLookAR();
+        let buttonText = "View in AR";
         // we can immediately enter this scene, because the platform supports rel="ar" links
-        if (supportsQuickLook) {
-            button.innerText = "View in AR";
+        if (DeviceUtilities.isVisionOS()) {
+            buttonText = "View in AR";
         }
-        else {
-            button.innerText = "View in AR";
+        else if (supportsQuickLook || DeviceUtilities.isiOS()) {
+            buttonText = "Open in Quicklook";
         }
+
+        button.innerText = buttonText;
         button.prepend(getIconElement("view_in_ar"));
 
         let createdExporter = false;
@@ -209,13 +212,20 @@ export class WebXRButtonFactory {
     }
 
     private updateSessionSupported(button: HTMLButtonElement, mode: XRSessionMode) {
+        if (mode === "immersive-ar" && DeviceUtilities.isiOS() && !DeviceUtilities.isVisionOS()) {
+            // on iOS, we can forward to the AppClip experience from the same button,
+            // so we always show the button. No AppClip support on VisionOS for now, so
+            // button state depends on WebXR support there.
+            return;
+        }
+
         if (!("xr" in navigator)) {
             button.style.display = "none";
             return;
         }
         NeedleXRSession.isSessionSupported(mode).then(supported => {
             button.style.display = !supported ? "none" : "";
-            if (isDevEnvironment() && !supported) console.log("[WebXR] \"" + mode + "\" is not supported on this device – make sure your server runs using HTTPS and you have a device connected that supports " + mode);
+            if (isDevEnvironment() && !supported) console.log("[WebXR] \"" + mode + "\" is not supported on this device. Make sure your server runs using HTTPS and you have a device connected that supports " + mode);
         });
     }
 

package/src/engine/webcomponents/needle menu/needle-menu.ts

@@ -1,5 +1,5 @@
 import type { Context } from "../../engine_context.js";
-import { hasCommercialLicense, onLicenseCheckResultChanged } from "../../engine_license.js";
+import { hasCommercialLicense, onLicenseCheckResultChanged, Telemetry } from "../../engine_license.js";
 import { isLocalNetwork } from "../../engine_networking_utils.js";
 import { DeviceUtilities, getParam } from "../../engine_utils.js";
 import { onXRSessionStart, XRSessionEventArgs } from "../../xr/events.js";
@@ -149,6 +149,9 @@ export class NeedleMenu {
                             else console.error("NeedleMenu: onclick is not a valid link", buttoninfo.onclick);
                         }
                     }
+                    Telemetry.sendEvent(this._context, "needle-menu", {
+                        action: "button_added_via_postmessage",
+                    });
                     this._menu.appendChild(button);
                 }
                 else if (debug) console.error("NeedleMenu: unknown postMessage event", data);
@@ -935,12 +938,39 @@ export class NeedleMenuElement extends HTMLElement {
     /** @private foldout container used in compact mode */
     private readonly foldout: HTMLDivElement;
 
+
+    private readonly trackedElements: WeakSet<Node> = new WeakSet();
+    private trackElement(el: Node) {
+        if (this.trackedElements.has(el)) return;
+        this.trackedElements.add(el);
+        el.addEventListener("click", (evt) => {
+            Telemetry.sendEvent(this._context, "needle-menu", {
+                action: "button_clicked",
+                element: evt.target instanceof Node ? evt.target.nodeName : el.nodeName,
+                label: el.textContent,
+                title: (el instanceof HTMLElement) ? el.title : undefined,
+                pointerid: (evt instanceof PointerEvent) ? evt.pointerId : undefined,
+            });
+        });
+        // el.addEventListener("pointerenter", (evt) => {
+        //     Telemetry.sendEvent(this._context, "needle-menu", {
+        //         action: "button_hovered",
+        //         element: evt.target instanceof Node ? evt.target.nodeName : el.nodeName,
+        //         label: el.textContent,
+        //         title: (el instanceof HTMLElement) ? el.title : undefined,
+        //         pointerid: (evt instanceof PointerEvent) ? evt.pointerId : undefined,
+        //     });
+        // });
+    }
+
     append(...nodes: (string | Node)[]): void {
         for (const node of nodes) {
             if (typeof node === "string") {
                 const element = document.createTextNode(node);
+                this.trackElement(element);
                 this.options.appendChild(element);
             } else {
+                this.trackElement(node);
                 this.options.appendChild(node);
             }
         }
@@ -987,9 +1017,10 @@ export class NeedleMenuElement extends HTMLElement {
             if (node.class) {
                 button.classList.add(node.class);
             }
+
             node = button as unknown as T;
         }
-
+        this.trackElement(node);
         const res = this.options.appendChild(node);
         return res;
     }
@@ -997,8 +1028,10 @@ export class NeedleMenuElement extends HTMLElement {
         for (const node of nodes) {
             if (typeof node === "string") {
                 const element = document.createTextNode(node);
+                this.trackElement(element);
                 this.options.prepend(element);
             } else {
+                this.trackElement(node);
                 this.options.prepend(node);
             }
         }

package/src/engine/webcomponents/needle-engine.ar-overlay.ts

@@ -118,6 +118,12 @@ export class AROverlayHandler {
     }
 
     private ensureQuitARButton(element: HTMLElement) {
+
+        // No quit button in app clips, we provide one via the native UI
+        if (DeviceUtilities.isNeedleAppClip()) {
+            return;
+        }
+
         const quitARSlot = document.createElement("slot");
         quitARSlot.setAttribute("name", "quit-ar");
         this.appendElement(quitARSlot, element);