@needle-tools/engine 4.8.7 → 4.8.8-next.5537a55

package/src/engine-components/webxr/WebXRImageTracking.ts

@@ -1,11 +1,12 @@
-import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
+import { ImageBitmapLoader, Matrix4, Object3D, Quaternion, Vector3 } from "three";
 import { Object3DEventMap } from "three";
 
 import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
 import { AssetReference } from "../../engine/engine_addressables.js";
+import { Context } from "../../engine/engine_context.js";
 import { serializable } from "../../engine/engine_serialization.js";
 import { IGameObject } from "../../engine/engine_types.js";
-import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
+import { CircularBuffer, delay, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
 import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/api.js";
 import { IUSDExporterExtension } from "../../engine-components/export/usdz/Extension.js";
 import { imageToCanvas, USDObject, USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
@@ -147,90 +148,122 @@ export class WebXRImageTrackingModel {
 
 class ImageTrackingExtension implements IUSDExporterExtension {
 
+
+
+    readonly isImageTrackingExtension = true;
     get extensionName() { return "image-tracking"; }
 
-    private filename: string;
-    private widthInMeters: number;
-    private imageData: Uint8Array;
 
-    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
-        this.filename = filename;
-        this.imageData = imageData;
-        this.widthInMeters = widthInMeters;
+    constructor(private readonly exporter: USDZExporter, private readonly component: WebXRImageTracking) {
+        if (debug) console.log(this);
+        this.exporter.anchoringType = "image";
+    }
+
+    // set during export
+    private shouldExport: boolean = true;
+
+    private filename: string = "marker.png";
+    private imageModel: WebXRImageTrackingModel | null = null;
+
+    onBeforeBuildDocument(_context: USDZExporterContext) {
+
+        // check if this extension is the first image tracking extension in the list
+        // since iOS can only track one image at a time we only allow one image tracking extension to be active
+        // we have to determine this at the earlierst export callback
+        // all subsequent export callbacks should then check is shouldExport is set to true
+        // this should only be the case for exactly one extension
+        const index = this.exporter.extensions
+            .filter(e => {
+                const ext = (e as ImageTrackingExtension);
+                return ext.isImageTrackingExtension && ext.component.activeAndEnabled && ext.component.trackedImages?.length > 0;
+            })
+            .indexOf(this);
+        this.shouldExport = index === 0;
+        if (!this.shouldExport) return;
+
+        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
+        if (this.component.trackedImages?.length > 1) {
+            if (debug || isDevEnvironment()) {
+                showBalloonWarning("USDZ: Only one tracked image is supported.");
+                console.warn("USDZ: Only one tracked image is supported. Will choose the first one in the trackedImages list");
+            }
+        }
     }
 
     onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
+        if (!this.shouldExport) return;
+
         const iOSVersion = DeviceUtilities.getiOSVersion();
         const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
         const workaroundForFB16119331 = majorVersion >= 18;
         const multiplier = workaroundForFB16119331 ? 1 : 100;
         writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
         writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
-        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
+        writer.appendLine(`uniform double physicalWidth = ` + (this.imageModel!.widthInMeters * multiplier).toFixed(8));
         writer.closeBlock();
+
     }
 
-    onBeforeBuildDocument(_context: USDZExporterContext) {
-        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
-        if (!imageTracking || !imageTracking.trackedImages) return;
+    async onAfterSerialize(context: USDZExporterContext) {
+        if (!this.shouldExport) return;
 
-        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
-        if (imageTracking.trackedImages.length > 1) 
-        {
-            if (isDevEnvironment()) showBalloonWarning("USDZ: Only one tracked image is supported.");
-            console.warn("USDZ: Only one tracked image is supported.");
-        }
-    }
+        const imageModel = this.imageModel;
+        const img = _imageElements.get(imageModel!.image!)!;
 
-    onAfterSerialize(context: USDZExporterContext) {
-        context.files['image_tracking/' + this.filename] = this.imageData;
+        const canvas = await imageToCanvas(img);
+        const blob = await canvas.convertToBlob({ type: 'image/png' });
+        const arrayBuffer = await blob.arrayBuffer();
+        context.files['image_tracking/' + this.filename] = new Uint8Array(arrayBuffer);
     }
 
     onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, _context: USDZExporterContext) {
-        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
-        if (!imageTracking || !imageTracking.trackedImages) return;
+        if (!this.shouldExport) return;
 
+        const imageTracking = this.component;
+        if (!imageTracking || !imageTracking.trackedImages?.length || !imageTracking.activeAndEnabled) return;
 
-        for (const trackedImage of imageTracking.trackedImages) {
-            if (trackedImage.object?.asset === object) {
-                const exporter = GameObject.findObjectOfType(USDZExporter);
-                if (!exporter) continue;
+        // we only care about the first image
+        const trackedImage = imageTracking.trackedImages[0];
 
-                const { scale, target } = exporter.getARScaleAndTarget();
+        if (trackedImage.object?.asset === object) {
+            this.imageModel = trackedImage;
 
-                // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
-                // On Android WebXR they're replaced by the tracked data
-                let parent = object;
-                const relativeMatrix = new Matrix4();
-                if (object !== target) {
-                    while (parent.parent && parent.parent !== target) {
-                        parent = parent.parent;
-                        relativeMatrix.premultiply(parent.matrix);
-                    }
+            const { scale, target } = this.exporter.getARScaleAndTarget();
+
+            // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
+            // On Android WebXR they're replaced by the tracked data
+            let parent = object;
+
+            const relativeMatrix = new Matrix4();
+            if (object !== target) {
+                while (parent && parent.parent && parent.parent !== target) {
+                    parent = parent.parent;
+                    relativeMatrix.premultiply(parent.matrix);
                 }
-                const mat = relativeMatrix
-                    .clone()
-                    .invert()
-                // apply session root scale again after undoing the world transformation
-                model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
-
-                // Unfortunately looks like Apple's docs are incomplete:
-                // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
-                // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
-                // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
-                // model.extraSchemas.push("Preliminary_AnchoringAPI");
-                // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
-                    // writer.appendLine( `token preliminary:anchoring:type = "image"` );
-                    // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
-                // });
-
-                // We can only apply this to the first tracked image, more are not supported by QuickLook.
-                break;
             }
+            const mat = relativeMatrix
+                .clone()
+                .invert()
+            // apply session root scale again after undoing the world transformation
+            model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
+
+
+            // Unfortunately looks like Apple's docs are incomplete:
+            // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
+            // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
+            // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
+            // model.extraSchemas.push("Preliminary_AnchoringAPI");
+            // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
+            // writer.appendLine( `token preliminary:anchoring:type = "image"` );
+            // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
+            // });
+
+            // We can only apply this to the first tracked image, more are not supported by QuickLook.
         }
     }
 }
 
+
 /**
  * @category XR
  * @group Components
@@ -238,14 +271,13 @@ class ImageTrackingExtension implements IUSDExporterExtension {
 export class WebXRImageTracking extends Behaviour {
 
     @serializable(WebXRImageTrackingModel)
-    trackedImages?: WebXRImageTrackingModel[];
+    trackedImages: WebXRImageTrackingModel[] = [];
 
     /** Applies smoothing based on detected jitter to the tracked image. */
     @serializable()
     smooth: boolean = true;
 
     private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
-    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
 
     /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
     get supported() { return this._supported; }
@@ -257,38 +289,24 @@ export class WebXRImageTracking extends Behaviour {
         if (!this.trackedImages) return;
         for (const trackedImage of this.trackedImages) {
             if (trackedImage.image) {
-                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
-                    // already loaded
-                }
-                else {
-                    const url = trackedImage.image;
-                    WebXRImageTracking._imageElements.set(url, null);
-                    const imageElement = document.createElement("img") as HTMLImageElement;
-                    imageElement.src = url;
-                    imageElement.addEventListener("load", async () => {
-                        const img = await createImageBitmap(imageElement);
-                        WebXRImageTracking._imageElements.set(url, img);
-
-                        // read back Uint8Array to use in USDZ - 
-                        // TODO better would be to do that once we actually need it
-                        const canvas = await imageToCanvas(img);
-                        if (canvas) {
-                            const blob = await canvas.convertToBlob({ type: 'image/png' });
-                            const arrayBuffer = await blob.arrayBuffer();
-
-                            const exporter = GameObject.findObjectOfType(USDZExporter);
-                            if (exporter && this.trackedImages) {
-                                exporter.extensions.push(
-                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
-                                );
-                                exporter.anchoringType = "image";
-                            }
-                        }
-                    });
-                }
+                loadImage(trackedImage.image);
             }
         }
     }
+    onEnable() {
+        USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport);
+    }
+    onDisable(): void {
+        USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport);
+    }
+
+    private onBeforeUSDZExport = (args: { exporter: USDZExporter }) => {
+        if (this.activeAndEnabled && this.trackedImages?.length) {
+            args.exporter.extensions.push(new ImageTrackingExtension(args.exporter, this));
+        }
+    }
+
+
 
     onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array<any> }): void {
         // console.log("onXRRequested", args, this.trackedImages)
@@ -300,7 +318,7 @@ export class WebXRImageTracking extends Behaviour {
             args.trackedImages = [];
             for (const trackedImage of this.trackedImages) {
                 if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
-                    const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
+                    const bitmap = _imageElements.get(trackedImage.image);
                     if (bitmap) {
                         this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
                         args.trackedImages.push({
@@ -337,7 +355,7 @@ export class WebXRImageTracking extends Behaviour {
 
     onLeaveXR(_args: NeedleXREventArgs): void {
 
-        if(!this.supported && DeviceUtilities.isAndroidDevice()) {
+        if (!this.supported && DeviceUtilities.isAndroidDevice()) {
             showBalloonWarning(this.webXRIncubationsWarning);
         }
 
@@ -364,7 +382,7 @@ export class WebXRImageTracking extends Behaviour {
     private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: Object3D | null, frames: number, lastTrackingTime: number }>();
     private readonly currentImages: WebXRTrackedImage[] = [];
 
-    
+
     private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
 
     onUpdateXR(args: NeedleXREventArgs): void {
@@ -517,4 +535,34 @@ export class WebXRImageTracking extends Behaviour {
             }
         }
     }
+}
+
+
+
+
+const _imageElements: Map<string, ImageBitmap | null> = new Map();
+const _imageLoadingPromises: Map<string, Promise<boolean>> = new Map();
+
+async function loadImage(url: string) {
+    if (_imageElements.has(url)) {
+        if (_imageLoadingPromises.has(url)) return _imageLoadingPromises.get(url);
+        return Promise.resolve(true);
+    }
+    const promise = new Promise<boolean>(res => {
+        _imageElements.set(url, null);
+        const imageElement = document.createElement("img") as HTMLImageElement;
+        imageElement.src = url;
+        imageElement.addEventListener("load", async () => {
+            const img = await createImageBitmap(imageElement);
+            _imageElements.set(url, img);
+            res(true);
+        });
+    });
+
+    _imageLoadingPromises.set(url, promise);
+    promise.finally(() => {
+        _imageLoadingPromises.delete(url);
+    });
+
+    return promise;
 }