@needle-tools/engine 4.7.0-alpha → 4.7.0-next.da47826

package/lib/engine-components/webxr/WebXRImageTracking.js

@@ -1,472 +1,472 @@
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-import { Matrix4, Quaternion, Vector3 } from "three";
-import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
-import { AssetReference } from "../../engine/engine_addressables.js";
-import { serializable } from "../../engine/engine_serialization.js";
-import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
-import { NeedleXRSession } from "../../engine/xr/api.js";
-import { imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
-import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
-import { Behaviour, GameObject } from "../Component.js";
-import { Renderer } from "../Renderer.js";
-// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
-const debug = getParam("debugimagetracking");
-export class WebXRTrackedImage {
-    get url() { return this._trackedImage.image ?? ""; }
-    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
-    get bitmap() { return this._bitmap; }
-    get model() { return this._trackedImage; }
-    measuredSize;
-    state;
-    /** Copy the image position to a vector */
-    getPosition(vec) {
-        this.ensureTransformData();
-        vec.copy(this._position);
-        return vec;
-    }
-    /** Copy the image rotation to a quaternion */
-    getQuaternion(quat) {
-        this.ensureTransformData();
-        quat.copy(this._rotation);
-        return quat;
-    }
-    applyToObject(object, t01 = undefined) {
-        this.ensureTransformData();
-        // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly
-        const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05;
-        if (t01)
-            t01 *= Math.max(1, changeAmount);
-        if (t01 === undefined || t01 >= 1) {
-            object.position.copy(this._position);
-            object.quaternion.copy(this._rotation);
-            // InstancingUtil.markDirty(object);
-        }
-        else {
-            t01 = Math.max(0, Math.min(1, t01));
-            object.position.lerp(this._position, t01);
-            object.quaternion.slerp(this._rotation, t01);
-            // InstancingUtil.markDirty(object);
-        }
-    }
-    static _positionBuffer = new CircularBuffer(() => new Vector3(), 20);
-    static _rotationBuffer = new CircularBuffer(() => new Quaternion(), 20);
-    _position;
-    _rotation;
-    ensureTransformData() {
-        if (!this._position) {
-            this._position = WebXRTrackedImage._positionBuffer.get();
-            this._rotation = WebXRTrackedImage._rotationBuffer.get();
-            const t = this._pose.transform;
-            const converted = NeedleXRSession.active.convertSpace(t);
-            this._position.copy(converted?.position);
-            this._rotation.copy(converted?.quaternion);
-        }
-    }
-    _trackingComponent;
-    _trackedImage;
-    _bitmap;
-    _pose;
-    constructor(context, trackedImage, bitmap, measuredSize, state, pose) {
-        this._trackingComponent = context;
-        ;
-        this._trackedImage = trackedImage;
-        this._bitmap = bitmap;
-        this.measuredSize = measuredSize;
-        this.state = state;
-        this._pose = pose;
-    }
-}
-/**
- * WebXRImageTracking allows you to track images in the real world and place objects on top of them.
- * This component is only available in WebXR sessions.
- * The WebXRImageTrackingModel contains the image to track, the object to place on top of the image, and the size of the image as well as settings for the tracking.
- * Used by the {@link WebXRImageTracking} component
- */
-export class WebXRImageTrackingModel {
-    /**
-     * Tracked image marker url. Make sure the image has good contrast and unique features to improve the tracking quality.
-     */
-    image;
-    /** Make sure this matches your physical marker size! Otherwise the tracked object will \"swim\" above or below the marker.
-     * @default 0.25 which is equivalent to 25cm
-    */
-    widthInMeters = .25;
-    /**
-     * The object moved around by the image. Make sure the size matches WidthInMeters.
-     */
-    object;
-    /**
-     * If true, a new instance of the referenced object will be created for each tracked image. Enable this if you're re-using objects for multiple markers.
-     */
-    createObjectInstance = false;
-    /** Use this for static images (e.g. markers on the floor). Only the first few frames of new poses will be applied to the model. This will result in more stable tracking.
-     * @default false
-    */
-    imageDoesNotMove = false;
-    /**
-     * Enable to hide the tracked object when the image is not tracked anymore. When disabled the tracked object will stay at the position it was last tracked at.
-     * @default true
-     */
-    hideWhenTrackingIsLost = true;
-}
-__decorate([
-    serializable(URL)
-], WebXRImageTrackingModel.prototype, "image", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "widthInMeters", void 0);
-__decorate([
-    serializable(AssetReference)
-], WebXRImageTrackingModel.prototype, "object", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "createObjectInstance", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "imageDoesNotMove", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "hideWhenTrackingIsLost", void 0);
-class ImageTrackingExtension {
-    get extensionName() { return "image-tracking"; }
-    filename;
-    widthInMeters;
-    imageData;
-    constructor(filename, imageData, widthInMeters) {
-        this.filename = filename;
-        this.imageData = imageData;
-        this.widthInMeters = widthInMeters;
-    }
-    onAfterHierarchy(_context, writer) {
-        const iOSVersion = DeviceUtilities.getiOSVersion();
-        const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
-        const workaroundForFB16119331 = majorVersion >= 18;
-        const multiplier = workaroundForFB16119331 ? 1 : 100;
-        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
-        writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
-        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
-        writer.closeBlock();
-    }
-    onBeforeBuildDocument(_context) {
-        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
-        if (!imageTracking || !imageTracking.trackedImages)
-            return;
-        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
-        if (imageTracking.trackedImages.length > 1) {
-            if (isDevEnvironment())
-                showBalloonWarning("USDZ: Only one tracked image is supported.");
-            console.warn("USDZ: Only one tracked image is supported.");
-        }
-    }
-    onAfterSerialize(context) {
-        context.files['image_tracking/' + this.filename] = this.imageData;
-    }
-    onExportObject(object, model, _context) {
-        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
-        if (!imageTracking || !imageTracking.trackedImages)
-            return;
-        for (const trackedImage of imageTracking.trackedImages) {
-            if (trackedImage.object?.asset === object) {
-                const exporter = GameObject.findObjectOfType(USDZExporter);
-                if (!exporter)
-                    continue;
-                const { scale, target } = exporter.getARScaleAndTarget();
-                // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
-                // On Android WebXR they're replaced by the tracked data
-                let parent = object;
-                const relativeMatrix = new Matrix4();
-                if (object !== target) {
-                    while (parent.parent && parent.parent !== target) {
-                        parent = parent.parent;
-                        relativeMatrix.premultiply(parent.matrix);
-                    }
-                }
-                const mat = relativeMatrix
-                    .clone()
-                    .invert();
-                // apply session root scale again after undoing the world transformation
-                model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
-                // Unfortunately looks like Apple's docs are incomplete:
-                // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
-                // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
-                // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
-                // model.extraSchemas.push("Preliminary_AnchoringAPI");
-                // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
-                // writer.appendLine( `token preliminary:anchoring:type = "image"` );
-                // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
-                // });
-                // We can only apply this to the first tracked image, more are not supported by QuickLook.
-                break;
-            }
-        }
-    }
-}
-/**
- * @category XR
- * @group Components
- */
-export class WebXRImageTracking extends Behaviour {
-    trackedImages;
-    /** Applies smoothing based on detected jitter to the tracked image. */
-    smooth = true;
-    trackedImageIndexMap = new Map();
-    static _imageElements = new Map();
-    /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
-    get supported() { return this._supported; }
-    _supported = true;
-    awake() {
-        if (debug)
-            console.log(this);
-        if (!this.trackedImages)
-            return;
-        for (const trackedImage of this.trackedImages) {
-            if (trackedImage.image) {
-                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
-                    // already loaded
-                }
-                else {
-                    const url = trackedImage.image;
-                    WebXRImageTracking._imageElements.set(url, null);
-                    const imageElement = document.createElement("img");
-                    imageElement.src = url;
-                    imageElement.addEventListener("load", async () => {
-                        const img = await createImageBitmap(imageElement);
-                        WebXRImageTracking._imageElements.set(url, img);
-                        // read back Uint8Array to use in USDZ - 
-                        // TODO better would be to do that once we actually need it
-                        const canvas = await imageToCanvas(img);
-                        if (canvas) {
-                            const blob = await canvas.convertToBlob({ type: 'image/png' });
-                            const arrayBuffer = await blob.arrayBuffer();
-                            const exporter = GameObject.findObjectOfType(USDZExporter);
-                            if (exporter && this.trackedImages) {
-                                exporter.extensions.push(new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters));
-                                exporter.anchoringType = "image";
-                            }
-                        }
-                    });
-                }
-            }
-        }
-    }
-    onBeforeXR(_mode, args) {
-        // console.log("onXRRequested", args, this.trackedImages)
-        if (this.trackedImages) {
-            args.optionalFeatures = args.optionalFeatures || [];
-            if (!args.optionalFeatures.includes("image-tracking"))
-                args.optionalFeatures.push("image-tracking");
-            args.trackedImages = [];
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
-                    const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
-                    if (bitmap) {
-                        this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
-                        args.trackedImages.push({
-                            image: bitmap,
-                            widthInMeters: trackedImage.widthInMeters
-                        });
-                    }
-                }
-            }
-        }
-    }
-    onEnterXR(_args) {
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    // capture the initial state of tracked images in the scene to restore them when the session ends
-                    const obj = trackedImage.object.asset;
-                    if (!obj.userData)
-                        obj.userData = {};
-                    const state = {
-                        visible: obj.visible,
-                        parent: obj.parent,
-                        matrix: obj.matrix.clone()
-                    };
-                    obj.userData["image-tracking"] = state;
-                }
-            }
-        }
-        // clear out all frame counters for tracking
-        for (const trackedData of this.imageToObjectMap.values()) {
-            trackedData.frames = 0;
-        }
-    }
-    ;
-    onLeaveXR(_args) {
-        if (!this.supported && DeviceUtilities.isAndroidDevice()) {
-            showBalloonWarning(this.webXRIncubationsWarning);
-        }
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    const obj = trackedImage.object.asset;
-                    if (obj.userData) {
-                        // restore the initial state of tracked images in the scene
-                        const state = obj.userData["image-tracking"];
-                        if (state) {
-                            obj.visible = state.visible;
-                            state.parent?.add(obj);
-                            obj.matrix.copy(state.matrix);
-                            obj.matrix.decompose(obj.position, obj.quaternion, obj.scale);
-                        }
-                        delete obj.userData["image-tracking"];
-                    }
-                }
-            }
-        }
-    }
-    imageToObjectMap = new Map();
-    currentImages = [];
-    webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
-    onUpdateXR(args) {
-        this.currentImages.length = 0;
-        const frame = args.xr.frame;
-        if (!frame)
-            return;
-        if (!("getImageTrackingResults" in frame)) {
-            if (!this["didPrintWarning"]) {
-                this["didPrintWarning"] = true;
-                console.log(this.webXRIncubationsWarning);
-            }
-            this._supported = false;
-            showBalloonWarning(this.webXRIncubationsWarning);
-            return;
-        }
-        // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch
-        // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled
-        else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) {
-            // Image tracking is not enabled for this session
-            return;
-        }
-        else if (frame.session && typeof frame.getImageTrackingResults === "function") {
-            const results = frame.getImageTrackingResults();
-            if (results.length > 0) {
-                const space = this.context.renderer.xr.getReferenceSpace();
-                if (space) {
-                    for (const result of results) {
-                        const state = result.trackingState;
-                        const imageIndex = result.index;
-                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
-                        if (trackedImage) {
-                            const pose = frame.getPose(result.imageSpace, space);
-                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
-                            this.currentImages.push(imageData);
-                        }
-                        else {
-                            if (debug) {
-                                console.warn("No tracked image for index", imageIndex);
-                            }
-                        }
-                    }
-                    if (this.currentImages.length > 0) {
-                        try {
-                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
-                            this.onImageTrackingUpdate(this.currentImages);
-                        }
-                        catch (e) {
-                            console.error(e);
-                        }
-                    }
-                }
-            }
-        }
-        // disable any objects that are no longer tracked
-        /** time in millis */
-        const hysteresis = 1000;
-        for (const [key, value] of this.imageToObjectMap) {
-            if (!value.object || !key)
-                continue;
-            // If the user disallowed hiding the object when tracking is lost, skip this
-            if (key.hideWhenTrackingIsLost === false)
-                continue;
-            let found = false;
-            for (const trackedImage of this.currentImages) {
-                if (trackedImage.model === key) {
-                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
-                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
-                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
-                        found = true;
-                        break;
-                    }
-                }
-            }
-            if (!found) {
-                GameObject.setActive(value.object, false);
-            }
-        }
-    }
-    onImageTrackingUpdate = (images) => {
-        const xr = NeedleXRSession.active;
-        if (!xr)
-            return;
-        for (const image of images) {
-            const model = image.model;
-            const isTracked = image.state === "tracked";
-            // don't do anything if we don't have an object to track - can be handled externally through events
-            if (!model.object)
-                continue;
-            let trackedData = this.imageToObjectMap.get(model);
-            if (trackedData === undefined) {
-                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
-                this.imageToObjectMap.set(model, trackedData);
-                model.object.loadAssetAsync().then((asset) => {
-                    if (model.createObjectInstance && asset) {
-                        asset = GameObject.instantiate(asset);
-                    }
-                    if (asset) {
-                        trackedData.object = asset;
-                        // workaround for instancing currently not properly updating 
-                        // instanced objects become visible when the image is recognized for the second time
-                        // we need to look into this further https://linear.app/needle/issue/NE-3936
-                        for (const rend of asset.getComponentsInChildren(Renderer)) {
-                            rend.setInstancingEnabled(false);
-                        }
-                        // make sure to parent to the WebXR.rig
-                        if (xr.rig) {
-                            xr.rig.gameObject.add(asset);
-                            image.applyToObject(asset);
-                            if (!asset.activeSelf)
-                                GameObject.setActive(asset, true);
-                            // InstancingUtil.markDirty(asset);
-                        }
-                        else {
-                            console.warn("XRImageTracking: missing XRRig");
-                        }
-                    }
-                });
-            }
-            else {
-                trackedData.frames++;
-                if (isTracked)
-                    trackedData.lastTrackingTime = Date.now();
-                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
-                // to improve the tracking quality a bit.
-                if (model.imageDoesNotMove && trackedData.frames > 10)
-                    continue;
-                if (!trackedData.object)
-                    continue;
-                if (xr.rig) {
-                    xr.rig.gameObject.add(trackedData.object);
-                    image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined);
-                    if (!trackedData.object.activeSelf) {
-                        GameObject.setActive(trackedData.object, true);
-                    }
-                    // InstancingUtil.markDirty(trackedData.object);
-                }
-            }
-        }
-    };
-}
-__decorate([
-    serializable(WebXRImageTrackingModel)
-], WebXRImageTracking.prototype, "trackedImages", void 0);
-__decorate([
-    serializable()
-], WebXRImageTracking.prototype, "smooth", void 0);
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { Matrix4, Quaternion, Vector3 } from "three";
+import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
+import { AssetReference } from "../../engine/engine_addressables.js";
+import { serializable } from "../../engine/engine_serialization.js";
+import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
+import { NeedleXRSession } from "../../engine/xr/api.js";
+import { imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
+import { Behaviour, GameObject } from "../Component.js";
+import { Renderer } from "../Renderer.js";
+// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
+const debug = getParam("debugimagetracking");
+export class WebXRTrackedImage {
+    get url() { return this._trackedImage.image ?? ""; }
+    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
+    get bitmap() { return this._bitmap; }
+    get model() { return this._trackedImage; }
+    measuredSize;
+    state;
+    /** Copy the image position to a vector */
+    getPosition(vec) {
+        this.ensureTransformData();
+        vec.copy(this._position);
+        return vec;
+    }
+    /** Copy the image rotation to a quaternion */
+    getQuaternion(quat) {
+        this.ensureTransformData();
+        quat.copy(this._rotation);
+        return quat;
+    }
+    applyToObject(object, t01 = undefined) {
+        this.ensureTransformData();
+        // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly
+        const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05;
+        if (t01)
+            t01 *= Math.max(1, changeAmount);
+        if (t01 === undefined || t01 >= 1) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+            // InstancingUtil.markDirty(object);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+            // InstancingUtil.markDirty(object);
+        }
+    }
+    static _positionBuffer = new CircularBuffer(() => new Vector3(), 20);
+    static _rotationBuffer = new CircularBuffer(() => new Quaternion(), 20);
+    _position;
+    _rotation;
+    ensureTransformData() {
+        if (!this._position) {
+            this._position = WebXRTrackedImage._positionBuffer.get();
+            this._rotation = WebXRTrackedImage._rotationBuffer.get();
+            const t = this._pose.transform;
+            const converted = NeedleXRSession.active.convertSpace(t);
+            this._position.copy(converted?.position);
+            this._rotation.copy(converted?.quaternion);
+        }
+    }
+    _trackingComponent;
+    _trackedImage;
+    _bitmap;
+    _pose;
+    constructor(context, trackedImage, bitmap, measuredSize, state, pose) {
+        this._trackingComponent = context;
+        ;
+        this._trackedImage = trackedImage;
+        this._bitmap = bitmap;
+        this.measuredSize = measuredSize;
+        this.state = state;
+        this._pose = pose;
+    }
+}
+/**
+ * WebXRImageTracking allows you to track images in the real world and place objects on top of them.
+ * This component is only available in WebXR sessions.
+ * The WebXRImageTrackingModel contains the image to track, the object to place on top of the image, and the size of the image as well as settings for the tracking.
+ * Used by the {@link WebXRImageTracking} component
+ */
+export class WebXRImageTrackingModel {
+    /**
+     * Tracked image marker url. Make sure the image has good contrast and unique features to improve the tracking quality.
+     */
+    image;
+    /** Make sure this matches your physical marker size! Otherwise the tracked object will \"swim\" above or below the marker.
+     * @default 0.25 which is equivalent to 25cm
+    */
+    widthInMeters = .25;
+    /**
+     * The object moved around by the image. Make sure the size matches WidthInMeters.
+     */
+    object;
+    /**
+     * If true, a new instance of the referenced object will be created for each tracked image. Enable this if you're re-using objects for multiple markers.
+     */
+    createObjectInstance = false;
+    /** Use this for static images (e.g. markers on the floor). Only the first few frames of new poses will be applied to the model. This will result in more stable tracking.
+     * @default false
+    */
+    imageDoesNotMove = false;
+    /**
+     * Enable to hide the tracked object when the image is not tracked anymore. When disabled the tracked object will stay at the position it was last tracked at.
+     * @default true
+     */
+    hideWhenTrackingIsLost = true;
+}
+__decorate([
+    serializable(URL)
+], WebXRImageTrackingModel.prototype, "image", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "widthInMeters", void 0);
+__decorate([
+    serializable(AssetReference)
+], WebXRImageTrackingModel.prototype, "object", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "createObjectInstance", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "imageDoesNotMove", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "hideWhenTrackingIsLost", void 0);
+class ImageTrackingExtension {
+    get extensionName() { return "image-tracking"; }
+    filename;
+    widthInMeters;
+    imageData;
+    constructor(filename, imageData, widthInMeters) {
+        this.filename = filename;
+        this.imageData = imageData;
+        this.widthInMeters = widthInMeters;
+    }
+    onAfterHierarchy(_context, writer) {
+        const iOSVersion = DeviceUtilities.getiOSVersion();
+        const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
+        const workaroundForFB16119331 = majorVersion >= 18;
+        const multiplier = workaroundForFB16119331 ? 1 : 100;
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
+        writer.closeBlock();
+    }
+    onBeforeBuildDocument(_context) {
+        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
+        if (!imageTracking || !imageTracking.trackedImages)
+            return;
+        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
+        if (imageTracking.trackedImages.length > 1) {
+            if (isDevEnvironment())
+                showBalloonWarning("USDZ: Only one tracked image is supported.");
+            console.warn("USDZ: Only one tracked image is supported.");
+        }
+    }
+    onAfterSerialize(context) {
+        context.files['image_tracking/' + this.filename] = this.imageData;
+    }
+    onExportObject(object, model, _context) {
+        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
+        if (!imageTracking || !imageTracking.trackedImages)
+            return;
+        for (const trackedImage of imageTracking.trackedImages) {
+            if (trackedImage.object?.asset === object) {
+                const exporter = GameObject.findObjectOfType(USDZExporter);
+                if (!exporter)
+                    continue;
+                const { scale, target } = exporter.getARScaleAndTarget();
+                // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
+                // On Android WebXR they're replaced by the tracked data
+                let parent = object;
+                const relativeMatrix = new Matrix4();
+                if (object !== target) {
+                    while (parent.parent && parent.parent !== target) {
+                        parent = parent.parent;
+                        relativeMatrix.premultiply(parent.matrix);
+                    }
+                }
+                const mat = relativeMatrix
+                    .clone()
+                    .invert();
+                // apply session root scale again after undoing the world transformation
+                model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
+                // Unfortunately looks like Apple's docs are incomplete:
+                // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
+                // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
+                // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
+                // model.extraSchemas.push("Preliminary_AnchoringAPI");
+                // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
+                // writer.appendLine( `token preliminary:anchoring:type = "image"` );
+                // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
+                // });
+                // We can only apply this to the first tracked image, more are not supported by QuickLook.
+                break;
+            }
+        }
+    }
+}
+/**
+ * @category XR
+ * @group Components
+ */
+export class WebXRImageTracking extends Behaviour {
+    trackedImages;
+    /** Applies smoothing based on detected jitter to the tracked image. */
+    smooth = true;
+    trackedImageIndexMap = new Map();
+    static _imageElements = new Map();
+    /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
+    get supported() { return this._supported; }
+    _supported = true;
+    awake() {
+        if (debug)
+            console.log(this);
+        if (!this.trackedImages)
+            return;
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image) {
+                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
+                    // already loaded
+                }
+                else {
+                    const url = trackedImage.image;
+                    WebXRImageTracking._imageElements.set(url, null);
+                    const imageElement = document.createElement("img");
+                    imageElement.src = url;
+                    imageElement.addEventListener("load", async () => {
+                        const img = await createImageBitmap(imageElement);
+                        WebXRImageTracking._imageElements.set(url, img);
+                        // read back Uint8Array to use in USDZ - 
+                        // TODO better would be to do that once we actually need it
+                        const canvas = await imageToCanvas(img);
+                        if (canvas) {
+                            const blob = await canvas.convertToBlob({ type: 'image/png' });
+                            const arrayBuffer = await blob.arrayBuffer();
+                            const exporter = GameObject.findObjectOfType(USDZExporter);
+                            if (exporter && this.trackedImages) {
+                                exporter.extensions.push(new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters));
+                                exporter.anchoringType = "image";
+                            }
+                        }
+                    });
+                }
+            }
+        }
+    }
+    onBeforeXR(_mode, args) {
+        // console.log("onXRRequested", args, this.trackedImages)
+        if (this.trackedImages) {
+            args.optionalFeatures = args.optionalFeatures || [];
+            if (!args.optionalFeatures.includes("image-tracking"))
+                args.optionalFeatures.push("image-tracking");
+            args.trackedImages = [];
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
+                    const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
+                    if (bitmap) {
+                        this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
+                        args.trackedImages.push({
+                            image: bitmap,
+                            widthInMeters: trackedImage.widthInMeters
+                        });
+                    }
+                }
+            }
+        }
+    }
+    onEnterXR(_args) {
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    // capture the initial state of tracked images in the scene to restore them when the session ends
+                    const obj = trackedImage.object.asset;
+                    if (!obj.userData)
+                        obj.userData = {};
+                    const state = {
+                        visible: obj.visible,
+                        parent: obj.parent,
+                        matrix: obj.matrix.clone()
+                    };
+                    obj.userData["image-tracking"] = state;
+                }
+            }
+        }
+        // clear out all frame counters for tracking
+        for (const trackedData of this.imageToObjectMap.values()) {
+            trackedData.frames = 0;
+        }
+    }
+    ;
+    onLeaveXR(_args) {
+        if (!this.supported && DeviceUtilities.isAndroidDevice()) {
+            showBalloonWarning(this.webXRIncubationsWarning);
+        }
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    const obj = trackedImage.object.asset;
+                    if (obj.userData) {
+                        // restore the initial state of tracked images in the scene
+                        const state = obj.userData["image-tracking"];
+                        if (state) {
+                            obj.visible = state.visible;
+                            state.parent?.add(obj);
+                            obj.matrix.copy(state.matrix);
+                            obj.matrix.decompose(obj.position, obj.quaternion, obj.scale);
+                        }
+                        delete obj.userData["image-tracking"];
+                    }
+                }
+            }
+        }
+    }
+    imageToObjectMap = new Map();
+    currentImages = [];
+    webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
+    onUpdateXR(args) {
+        this.currentImages.length = 0;
+        const frame = args.xr.frame;
+        if (!frame)
+            return;
+        if (!("getImageTrackingResults" in frame)) {
+            if (!this["didPrintWarning"]) {
+                this["didPrintWarning"] = true;
+                console.log(this.webXRIncubationsWarning);
+            }
+            this._supported = false;
+            showBalloonWarning(this.webXRIncubationsWarning);
+            return;
+        }
+        // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch
+        // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled
+        else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) {
+            // Image tracking is not enabled for this session
+            return;
+        }
+        else if (frame.session && typeof frame.getImageTrackingResults === "function") {
+            const results = frame.getImageTrackingResults();
+            if (results.length > 0) {
+                const space = this.context.renderer.xr.getReferenceSpace();
+                if (space) {
+                    for (const result of results) {
+                        const state = result.trackingState;
+                        const imageIndex = result.index;
+                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
+                        if (trackedImage) {
+                            const pose = frame.getPose(result.imageSpace, space);
+                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
+                            this.currentImages.push(imageData);
+                        }
+                        else {
+                            if (debug) {
+                                console.warn("No tracked image for index", imageIndex);
+                            }
+                        }
+                    }
+                    if (this.currentImages.length > 0) {
+                        try {
+                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
+                            this.onImageTrackingUpdate(this.currentImages);
+                        }
+                        catch (e) {
+                            console.error(e);
+                        }
+                    }
+                }
+            }
+        }
+        // disable any objects that are no longer tracked
+        /** time in millis */
+        const hysteresis = 1000;
+        for (const [key, value] of this.imageToObjectMap) {
+            if (!value.object || !key)
+                continue;
+            // If the user disallowed hiding the object when tracking is lost, skip this
+            if (key.hideWhenTrackingIsLost === false)
+                continue;
+            let found = false;
+            for (const trackedImage of this.currentImages) {
+                if (trackedImage.model === key) {
+                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
+                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
+                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            if (!found) {
+                GameObject.setActive(value.object, false);
+            }
+        }
+    }
+    onImageTrackingUpdate = (images) => {
+        const xr = NeedleXRSession.active;
+        if (!xr)
+            return;
+        for (const image of images) {
+            const model = image.model;
+            const isTracked = image.state === "tracked";
+            // don't do anything if we don't have an object to track - can be handled externally through events
+            if (!model.object)
+                continue;
+            let trackedData = this.imageToObjectMap.get(model);
+            if (trackedData === undefined) {
+                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
+                this.imageToObjectMap.set(model, trackedData);
+                model.object.loadAssetAsync().then((asset) => {
+                    if (model.createObjectInstance && asset) {
+                        asset = GameObject.instantiate(asset);
+                    }
+                    if (asset) {
+                        trackedData.object = asset;
+                        // workaround for instancing currently not properly updating 
+                        // instanced objects become visible when the image is recognized for the second time
+                        // we need to look into this further https://linear.app/needle/issue/NE-3936
+                        for (const rend of asset.getComponentsInChildren(Renderer)) {
+                            rend.setInstancingEnabled(false);
+                        }
+                        // make sure to parent to the WebXR.rig
+                        if (xr.rig) {
+                            xr.rig.gameObject.add(asset);
+                            image.applyToObject(asset);
+                            if (!asset.activeSelf)
+                                GameObject.setActive(asset, true);
+                            // InstancingUtil.markDirty(asset);
+                        }
+                        else {
+                            console.warn("XRImageTracking: missing XRRig");
+                        }
+                    }
+                });
+            }
+            else {
+                trackedData.frames++;
+                if (isTracked)
+                    trackedData.lastTrackingTime = Date.now();
+                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
+                // to improve the tracking quality a bit.
+                if (model.imageDoesNotMove && trackedData.frames > 10)
+                    continue;
+                if (!trackedData.object)
+                    continue;
+                if (xr.rig) {
+                    xr.rig.gameObject.add(trackedData.object);
+                    image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined);
+                    if (!trackedData.object.activeSelf) {
+                        GameObject.setActive(trackedData.object, true);
+                    }
+                    // InstancingUtil.markDirty(trackedData.object);
+                }
+            }
+        }
+    };
+}
+__decorate([
+    serializable(WebXRImageTrackingModel)
+], WebXRImageTracking.prototype, "trackedImages", void 0);
+__decorate([
+    serializable()
+], WebXRImageTracking.prototype, "smooth", void 0);
 //# sourceMappingURL=WebXRImageTracking.js.map