@needle-tools/engine 3.28.6-beta → 3.28.7-beta.1

package/src/engine-components/webxr/WebXRImageTracking.ts

@@ -1,372 +1,372 @@
-import { WebXR, WebXREvent } from "./WebXR.js";
-import { serializable } from "../../engine/engine_serialization.js";
-import { Behaviour, GameObject } from "../Component.js";
-import { Object3D, Quaternion, Vector3 } from "three";
-import { CircularBuffer, getParam } from "../../engine/engine_utils.js";
-import { AssetReference } from "../../engine/engine_addressables.js";
-import { showBalloonMessage, showBalloonWarning } from "../../engine/debug/index.js";
-
-import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
-import { USDZExporterContext, USDWriter, imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
-
-// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
-
-const debug = getParam("debugimagetracking");
-
-export class WebXRTrackedImage {
-
-    get url(): string { return this._trackedImage.image ?? ""; }
-    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
-    get bitmap(): ImageBitmap { return this._bitmap; }
-    get model(): WebXRImageTrackingModel { return this._trackedImage; }
-    readonly measuredSize: number;
-    readonly state: "tracked" | "emulated";
-
-    /** Copy the image position to a vector */
-    getPosition(vec: Vector3) {
-        this.ensureTransformData();
-        vec.copy(this._position);
-        return vec;
-    }
-
-    /** Copy the image rotation to a quaternion */
-    getQuaternion(quat: Quaternion) {
-        this.ensureTransformData();
-        quat.copy(this._rotation);
-        return quat;
-    }
-
-    private static y180 = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
-    applyToObject(object: Object3D, t01: number | undefined = undefined) {
-        this.ensureTransformData();
-        // check if position/_position or rotation/_rotation changed more than just a little bit
-        const haveChanged = object.position.distanceToSquared(this._position) > 0.05 || object.quaternion.angleTo(this._rotation) > 0.05;
-        if (t01 === undefined || t01 >= 1 || haveChanged) {
-            object.position.copy(this._position);
-            object.quaternion.copy(this._rotation);
-        }
-        else {
-            t01 = Math.max(0, Math.min(1, t01));
-            object.position.lerp(this._position, t01);
-            object.quaternion.slerp(this._rotation, t01);
-        }
-        object.quaternion.multiply(WebXRTrackedImage.y180);
-    }
-
-    private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
-    private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
-    private _position!: Vector3;
-    private _rotation!: Quaternion;
-    private ensureTransformData() {
-        if (!this._position) {
-            this._position = WebXRTrackedImage._positionBuffer.get();
-            this._rotation = WebXRTrackedImage._rotationBuffer.get();
-            const t = this._pose.transform;
-
-            // when parented to the world, we need to flip data here
-            //this._position.set(-t.position.x, t.position.y, -t.position.z);
-            // this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
-
-            // for some reason when parented to the XRRig, we need the original data
-            this._position.set(t.position.x, t.position.y, t.position.z);
-            this._rotation.set(t.orientation.x, t.orientation.y, t.orientation.z, t.orientation.w);
-        }
-    }
-
-    private readonly _trackingComponent: WebXRImageTracking;
-    private readonly _trackedImage: WebXRImageTrackingModel;
-    private readonly _bitmap: ImageBitmap;
-    private readonly _pose: any;
-
-    constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
-        this._trackingComponent = context;;
-        this._trackedImage = trackedImage;
-        this._bitmap = bitmap;
-        this.measuredSize = measuredSize;
-        this.state = state;
-        this._pose = pose;
-    }
-
-}
-
-declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
-
-export class WebXRImageTrackingModel {
-
-    @serializable(URL)
-    image?: string;
-
-    @serializable()
-    widthInMeters!: number;
-
-    @serializable(AssetReference)
-    object?: AssetReference;
-
-    @serializable()
-    createObjectInstance: boolean = false;
-
-    @serializable()
-    imageDoesNotMove: boolean = false;
-}
-
-class ImageTrackingExtension {
-
-    get extensionName() { return "image-tracking"; }
-
-    private filename: string;
-    private widthInMeters: number;
-    private imageData: Uint8Array;
-
-    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
-        this.filename = filename;
-        this.imageData = imageData;
-        this.widthInMeters = widthInMeters;
-    }
-
-    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
-        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
-        writer.appendLine(`uniform asset image = @tracker/` + this.filename + `@`);
-        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * 100).toFixed(8));
-        writer.closeBlock();
-    }
-
-    onAfterSerialize(context: USDZExporterContext) {
-        context.files['tracker/' + this.filename] = this.imageData;
-    }
-}
-
-export class WebXRImageTracking extends Behaviour {
-
-    @serializable(WebXRImageTrackingModel)
-    trackedImages?: WebXRImageTrackingModel[];
-
-    private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
-
-    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
-    private webxr: WebXR | null = null;
-
-    awake(): void {
-        if (debug) console.log(this)
-        if (!this.trackedImages) return;
-        for (const trackedImage of this.trackedImages) {
-            if (trackedImage.image) {
-                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
-                }
-                else {
-                    const url = trackedImage.image;
-                    WebXRImageTracking._imageElements.set(url, null);
-                    const imageElement = document.createElement("img") as HTMLImageElement;
-                    imageElement.src = url;
-                    imageElement.addEventListener("load", async () => {
-                        const img = await createImageBitmap(imageElement);
-                        WebXRImageTracking._imageElements.set(url, img);
-
-                        // read back Uint8Array to use in USDZ - 
-                        // TODO better would be to do that once we actually need it
-                        const canvas = await imageToCanvas(img);
-                        if (canvas) {
-                            const blob = await canvas.convertToBlob({ type: 'image/png' });
-                            const arrayBuffer = await blob.arrayBuffer();
-
-                            const exporter = GameObject.findObjectOfType(USDZExporter);
-                            if (exporter && this.trackedImages) {
-                                exporter.extensions.push(
-                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
-                                );
-                                exporter.anchoringType = "image";
-                            }
-                        }
-                    });
-                }
-            }
-        }
-    }
-
-
-    onEnable(): void {
-        this.webxr = GameObject.findObjectOfType(WebXR);
-        WebXR.addEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
-        WebXR.addEventListener(WebXREvent.XRStarted, this.onXRStarted);
-        WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
-        this.addEventListener("image-tracking", this.onImageTrackingUpdate);
-    }
-
-    onDisable(): void {
-        WebXR.removeEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
-        WebXR.removeEventListener(WebXREvent.XRStarted, this.onXRStarted);
-        WebXR.removeEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
-        this.removeEventListener("image-tracking", this.onImageTrackingUpdate);
-    }
-
-    private onModifyAROptions = (event: any) => {
-        if (!this.trackedImages) return;
-        const options = event.detail;
-        const features = options.optionalFeatures || [];
-        if (!features.includes("image-tracking"))
-            features.push("image-tracking");
-        options.optionalFeatures = features;
-
-        options.trackedImages = [];
-        for (const trackedImage of this.trackedImages) {
-            if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
-                const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
-                if (bitmap) {
-                    this.trackedImageIndexMap.set(options.trackedImages.length, trackedImage);
-                    options.trackedImages.push({
-                        image: bitmap,
-                        widthInMeters: trackedImage.widthInMeters
-                    });
-                }
-            }
-        }
-    }
-
-    private onXRStarted = (_: any) => {
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    const obj = trackedImage.object.asset;
-                    obj.visible = false;
-                }
-            }
-        }
-        // clear out all frame counters for tracking
-        for (const trackedData of this.imageToObjectMap.values()) {
-            trackedData.frames = 0;
-        }
-    };
-
-    private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: GameObject | null, frames: number, lastTrackingTime:number }>();
-    private readonly currentImages: WebXRTrackedImage[] = [];
-
-
-    private onXRUpdate = (evt): void => {
-        this.currentImages.length = 0;
-
-        const frame = evt.frame;
-        if (!frame) return;
-
-        if (frame.session && !("getImageTrackingResults" in frame)) {
-            const warning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
-            if (!this["didPrintWarning"]) {
-                this["didPrintWarning"] = true;
-                console.log(warning);
-            }
-            showBalloonWarning(warning);
-            return;
-        }
-
-        if (frame.session && typeof frame.getImageTrackingResults === "function") {
-            const results = frame.getImageTrackingResults();
-            if (results.length > 0) {
-                const space = this.context.renderer.xr.getReferenceSpace();
-                if (space) {
-                    for (const result of results) {
-                        const state = result.trackingState;
-                        const imageIndex = result.index;
-                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
-                        if (trackedImage) {
-                            const pose = frame.getPose(result.imageSpace, space);
-                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
-                            this.currentImages.push(imageData);
-                        }
-                        else {
-                            if (debug) {
-                                console.warn("No tracked image for index", imageIndex);
-                            }
-                        }
-                    }
-                    if (this.currentImages.length > 0) {
-                        try {
-                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
-                            if (this.webxr && this.webxr.allowARPlacementReticle) {
-                                this.webxr.allowARPlacementReticle = false;
-                            }
-                        }
-                        catch (e) {
-                            console.error(e);
-                        }
-                    }
-                }
-            }
-        }
-
-        // disable any objects that are no longer tracked
-        /** time in millis */
-        const hysteresis = 1000;
-        for (const [key, value] of this.imageToObjectMap) {
-            if (!value.object || !key) continue;
-            let found = false;
-            for (const trackedImage of this.currentImages) {
-                if (trackedImage.model === key) {
-                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
-                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
-                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
-                        found = true;
-                        break;
-                    }
-                }
-            }
-            if (!found) {
-                GameObject.setActive(value.object, false);
-            }
-        }
-    }
-
-
-    private onImageTrackingUpdate = (event: any) => {
-        const images = event.detail as WebXRTrackedImage[];
-
-        for (const image of images) {
-            const model = image.model;
-            const isTracked = image.state === "tracked";
-            // don't do anything if we don't have an object to track - can be handled externally through events
-            if (!model.object) continue;
-
-            let trackedData = this.imageToObjectMap.get(model);
-            if (trackedData === undefined) {
-                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
-                this.imageToObjectMap.set(model, trackedData);
-
-                model.object.loadAssetAsync().then((asset: GameObject | null) => {
-                    if (model.createObjectInstance) {
-                        asset = GameObject.instantiate(asset);
-                    }
-
-                    if (asset) {
-                        trackedData!.object = asset;
-
-                        // make sure to parent to the WebXR.rig
-                        if (this.webxr) {
-                            this.webxr.Rig.add(asset);
-                        }
-
-                        image.applyToObject(asset);
-                        if (!asset.activeSelf)
-                            GameObject.setActive(asset, true);
-                    }
-                });
-            }
-            else {
-                trackedData.frames++;
-                if(isTracked)
-                    trackedData.lastTrackingTime = Date.now();
-
-                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
-                // to improve the tracking quality a bit.
-                if (model.imageDoesNotMove && trackedData.frames > 10)
-                    continue;
-
-                if (!trackedData.object) continue;
-
-                if (this.webxr) {
-                    this.webxr.Rig.add(trackedData.object);
-                }
-
-                image.applyToObject(trackedData.object);
-                if (!trackedData.object.activeSelf)
-                    GameObject.setActive(trackedData.object, true);
-            }
-        }
-    }
+import { WebXR, WebXREvent } from "./WebXR.js";
+import { serializable } from "../../engine/engine_serialization.js";
+import { Behaviour, GameObject } from "../Component.js";
+import { Object3D, Quaternion, Vector3 } from "three";
+import { CircularBuffer, getParam } from "../../engine/engine_utils.js";
+import { AssetReference } from "../../engine/engine_addressables.js";
+import { showBalloonMessage, showBalloonWarning } from "../../engine/debug/index.js";
+
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
+import { USDZExporterContext, USDWriter, imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
+
+// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
+
+const debug = getParam("debugimagetracking");
+
+export class WebXRTrackedImage {
+
+    get url(): string { return this._trackedImage.image ?? ""; }
+    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
+    get bitmap(): ImageBitmap { return this._bitmap; }
+    get model(): WebXRImageTrackingModel { return this._trackedImage; }
+    readonly measuredSize: number;
+    readonly state: "tracked" | "emulated";
+
+    /** Copy the image position to a vector */
+    getPosition(vec: Vector3) {
+        this.ensureTransformData();
+        vec.copy(this._position);
+        return vec;
+    }
+
+    /** Copy the image rotation to a quaternion */
+    getQuaternion(quat: Quaternion) {
+        this.ensureTransformData();
+        quat.copy(this._rotation);
+        return quat;
+    }
+
+    private static y180 = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
+    applyToObject(object: Object3D, t01: number | undefined = undefined) {
+        this.ensureTransformData();
+        // check if position/_position or rotation/_rotation changed more than just a little bit
+        const haveChanged = object.position.distanceToSquared(this._position) > 0.05 || object.quaternion.angleTo(this._rotation) > 0.05;
+        if (t01 === undefined || t01 >= 1 || haveChanged) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+        }
+        object.quaternion.multiply(WebXRTrackedImage.y180);
+    }
+
+    private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
+    private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
+    private _position!: Vector3;
+    private _rotation!: Quaternion;
+    private ensureTransformData() {
+        if (!this._position) {
+            this._position = WebXRTrackedImage._positionBuffer.get();
+            this._rotation = WebXRTrackedImage._rotationBuffer.get();
+            const t = this._pose.transform;
+
+            // when parented to the world, we need to flip data here
+            //this._position.set(-t.position.x, t.position.y, -t.position.z);
+            // this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+
+            // for some reason when parented to the XRRig, we need the original data
+            this._position.set(t.position.x, t.position.y, t.position.z);
+            this._rotation.set(t.orientation.x, t.orientation.y, t.orientation.z, t.orientation.w);
+        }
+    }
+
+    private readonly _trackingComponent: WebXRImageTracking;
+    private readonly _trackedImage: WebXRImageTrackingModel;
+    private readonly _bitmap: ImageBitmap;
+    private readonly _pose: any;
+
+    constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
+        this._trackingComponent = context;;
+        this._trackedImage = trackedImage;
+        this._bitmap = bitmap;
+        this.measuredSize = measuredSize;
+        this.state = state;
+        this._pose = pose;
+    }
+
+}
+
+declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
+
+export class WebXRImageTrackingModel {
+
+    @serializable(URL)
+    image?: string;
+
+    @serializable()
+    widthInMeters!: number;
+
+    @serializable(AssetReference)
+    object?: AssetReference;
+
+    @serializable()
+    createObjectInstance: boolean = false;
+
+    @serializable()
+    imageDoesNotMove: boolean = false;
+}
+
+class ImageTrackingExtension {
+
+    get extensionName() { return "image-tracking"; }
+
+    private filename: string;
+    private widthInMeters: number;
+    private imageData: Uint8Array;
+
+    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
+        this.filename = filename;
+        this.imageData = imageData;
+        this.widthInMeters = widthInMeters;
+    }
+
+    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @tracker/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * 100).toFixed(8));
+        writer.closeBlock();
+    }
+
+    onAfterSerialize(context: USDZExporterContext) {
+        context.files['tracker/' + this.filename] = this.imageData;
+    }
+}
+
+export class WebXRImageTracking extends Behaviour {
+
+    @serializable(WebXRImageTrackingModel)
+    trackedImages?: WebXRImageTrackingModel[];
+
+    private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
+
+    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
+    private webxr: WebXR | null = null;
+
+    awake(): void {
+        if (debug) console.log(this)
+        if (!this.trackedImages) return;
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image) {
+                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
+                }
+                else {
+                    const url = trackedImage.image;
+                    WebXRImageTracking._imageElements.set(url, null);
+                    const imageElement = document.createElement("img") as HTMLImageElement;
+                    imageElement.src = url;
+                    imageElement.addEventListener("load", async () => {
+                        const img = await createImageBitmap(imageElement);
+                        WebXRImageTracking._imageElements.set(url, img);
+
+                        // read back Uint8Array to use in USDZ - 
+                        // TODO better would be to do that once we actually need it
+                        const canvas = await imageToCanvas(img);
+                        if (canvas) {
+                            const blob = await canvas.convertToBlob({ type: 'image/png' });
+                            const arrayBuffer = await blob.arrayBuffer();
+
+                            const exporter = GameObject.findObjectOfType(USDZExporter);
+                            if (exporter && this.trackedImages) {
+                                exporter.extensions.push(
+                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
+                                );
+                                exporter.anchoringType = "image";
+                            }
+                        }
+                    });
+                }
+            }
+        }
+    }
+
+
+    onEnable(): void {
+        this.webxr = GameObject.findObjectOfType(WebXR);
+        WebXR.addEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.addEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.addEventListener("image-tracking", this.onImageTrackingUpdate);
+    }
+
+    onDisable(): void {
+        WebXR.removeEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.removeEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.removeEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.removeEventListener("image-tracking", this.onImageTrackingUpdate);
+    }
+
+    private onModifyAROptions = (event: any) => {
+        if (!this.trackedImages) return;
+        const options = event.detail;
+        const features = options.optionalFeatures || [];
+        if (!features.includes("image-tracking"))
+            features.push("image-tracking");
+        options.optionalFeatures = features;
+
+        options.trackedImages = [];
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
+                const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
+                if (bitmap) {
+                    this.trackedImageIndexMap.set(options.trackedImages.length, trackedImage);
+                    options.trackedImages.push({
+                        image: bitmap,
+                        widthInMeters: trackedImage.widthInMeters
+                    });
+                }
+            }
+        }
+    }
+
+    private onXRStarted = (_: any) => {
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    const obj = trackedImage.object.asset;
+                    obj.visible = false;
+                }
+            }
+        }
+        // clear out all frame counters for tracking
+        for (const trackedData of this.imageToObjectMap.values()) {
+            trackedData.frames = 0;
+        }
+    };
+
+    private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: GameObject | null, frames: number, lastTrackingTime:number }>();
+    private readonly currentImages: WebXRTrackedImage[] = [];
+
+
+    private onXRUpdate = (evt): void => {
+        this.currentImages.length = 0;
+
+        const frame = evt.frame;
+        if (!frame) return;
+
+        if (frame.session && !("getImageTrackingResults" in frame)) {
+            const warning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
+            if (!this["didPrintWarning"]) {
+                this["didPrintWarning"] = true;
+                console.log(warning);
+            }
+            showBalloonWarning(warning);
+            return;
+        }
+
+        if (frame.session && typeof frame.getImageTrackingResults === "function") {
+            const results = frame.getImageTrackingResults();
+            if (results.length > 0) {
+                const space = this.context.renderer.xr.getReferenceSpace();
+                if (space) {
+                    for (const result of results) {
+                        const state = result.trackingState;
+                        const imageIndex = result.index;
+                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
+                        if (trackedImage) {
+                            const pose = frame.getPose(result.imageSpace, space);
+                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
+                            this.currentImages.push(imageData);
+                        }
+                        else {
+                            if (debug) {
+                                console.warn("No tracked image for index", imageIndex);
+                            }
+                        }
+                    }
+                    if (this.currentImages.length > 0) {
+                        try {
+                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
+                            if (this.webxr && this.webxr.allowARPlacementReticle) {
+                                this.webxr.allowARPlacementReticle = false;
+                            }
+                        }
+                        catch (e) {
+                            console.error(e);
+                        }
+                    }
+                }
+            }
+        }
+
+        // disable any objects that are no longer tracked
+        /** time in millis */
+        const hysteresis = 1000;
+        for (const [key, value] of this.imageToObjectMap) {
+            if (!value.object || !key) continue;
+            let found = false;
+            for (const trackedImage of this.currentImages) {
+                if (trackedImage.model === key) {
+                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
+                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
+                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            if (!found) {
+                GameObject.setActive(value.object, false);
+            }
+        }
+    }
+
+
+    private onImageTrackingUpdate = (event: any) => {
+        const images = event.detail as WebXRTrackedImage[];
+
+        for (const image of images) {
+            const model = image.model;
+            const isTracked = image.state === "tracked";
+            // don't do anything if we don't have an object to track - can be handled externally through events
+            if (!model.object) continue;
+
+            let trackedData = this.imageToObjectMap.get(model);
+            if (trackedData === undefined) {
+                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
+                this.imageToObjectMap.set(model, trackedData);
+
+                model.object.loadAssetAsync().then((asset: GameObject | null) => {
+                    if (model.createObjectInstance) {
+                        asset = GameObject.instantiate(asset);
+                    }
+
+                    if (asset) {
+                        trackedData!.object = asset;
+
+                        // make sure to parent to the WebXR.rig
+                        if (this.webxr) {
+                            this.webxr.Rig.add(asset);
+                        }
+
+                        image.applyToObject(asset);
+                        if (!asset.activeSelf)
+                            GameObject.setActive(asset, true);
+                    }
+                });
+            }
+            else {
+                trackedData.frames++;
+                if(isTracked)
+                    trackedData.lastTrackingTime = Date.now();
+
+                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
+                // to improve the tracking quality a bit.
+                if (model.imageDoesNotMove && trackedData.frames > 10)
+                    continue;
+
+                if (!trackedData.object) continue;
+
+                if (this.webxr) {
+                    this.webxr.Rig.add(trackedData.object);
+                }
+
+                image.applyToObject(trackedData.object);
+                if (!trackedData.object.activeSelf)
+                    GameObject.setActive(trackedData.object, true);
+            }
+        }
+    }
 }