@needle-tools/engine 4.7.0-alpha → 4.7.0-next.4ec920e

package/src/engine-components/webxr/WebXRImageTracking.ts

@@ -1,520 +1,520 @@
-import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
-import { Object3DEventMap } from "three";
-
-import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
-import { AssetReference } from "../../engine/engine_addressables.js";
-import { serializable } from "../../engine/engine_serialization.js";
-import { IGameObject } from "../../engine/engine_types.js";
-import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
-import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/api.js";
-import { IUSDExporterExtension } from "../../engine-components/export/usdz/Extension.js";
-import { imageToCanvas, USDObject, USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
-import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
-import { Behaviour, GameObject } from "../Component.js";
-import { Renderer } from "../Renderer.js";
-
-// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
-
-const debug = getParam("debugimagetracking");
-
-export class WebXRTrackedImage {
-
-    get url(): string { return this._trackedImage.image ?? ""; }
-    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
-    get bitmap(): ImageBitmap { return this._bitmap; }
-    get model(): WebXRImageTrackingModel { return this._trackedImage; }
-    readonly measuredSize: number;
-    readonly state: "tracked" | "emulated";
-
-    /** Copy the image position to a vector */
-    getPosition(vec: Vector3) {
-        this.ensureTransformData();
-        vec.copy(this._position);
-        return vec;
-    }
-
-    /** Copy the image rotation to a quaternion */
-    getQuaternion(quat: Quaternion) {
-        this.ensureTransformData();
-        quat.copy(this._rotation);
-        return quat;
-    }
-
-    applyToObject(object: Object3D, t01: number | undefined = undefined) {
-        this.ensureTransformData();
-        // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly
-        const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05;
-        if (t01) t01 *= Math.max(1, changeAmount);
-        if (t01 === undefined || t01 >= 1) {
-            object.position.copy(this._position);
-            object.quaternion.copy(this._rotation);
-            // InstancingUtil.markDirty(object);
-        }
-        else {
-            t01 = Math.max(0, Math.min(1, t01));
-            object.position.lerp(this._position, t01);
-            object.quaternion.slerp(this._rotation, t01);
-            // InstancingUtil.markDirty(object);
-        }
-    }
-
-    private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
-    private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
-    private _position!: Vector3;
-    private _rotation!: Quaternion;
-    private ensureTransformData() {
-        if (!this._position) {
-            this._position = WebXRTrackedImage._positionBuffer.get();
-            this._rotation = WebXRTrackedImage._rotationBuffer.get();
-            const t = this._pose.transform as XRRigidTransform;
-            const converted = NeedleXRSession.active!.convertSpace(t);
-            this._position.copy(converted?.position);
-            this._rotation.copy(converted?.quaternion);
-        }
-    }
-
-    private readonly _trackingComponent: WebXRImageTracking;
-    private readonly _trackedImage: WebXRImageTrackingModel;
-    private readonly _bitmap: ImageBitmap;
-    private readonly _pose: any;
-
-    constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
-        this._trackingComponent = context;;
-        this._trackedImage = trackedImage;
-        this._bitmap = bitmap;
-        this.measuredSize = measuredSize;
-        this.state = state;
-        this._pose = pose;
-    }
-
-}
-
-declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
-
-/** Initial state of assigned tracked image objects that are already present in the scene
- * This is used to restore the state of the object when the webxr session has ended to return to the original state
- */
-declare type InitialTrackedObjectState = {
-    visible: boolean;
-    parent: Object3D | undefined | null;
-    matrix: Matrix4;
-}
-/**
- * WebXRImageTracking allows you to track images in the real world and place objects on top of them.  
- * This component is only available in WebXR sessions.  
- * The WebXRImageTrackingModel contains the image to track, the object to place on top of the image, and the size of the image as well as settings for the tracking.  
- * Used by the {@link WebXRImageTracking} component
- */
-export class WebXRImageTrackingModel {
-
-    /**
-     * Tracked image marker url. Make sure the image has good contrast and unique features to improve the tracking quality.
-     */
-    @serializable(URL)
-    image?: string;
-
-    /** Make sure this matches your physical marker size! Otherwise the tracked object will \"swim\" above or below the marker. 
-     * @default 0.25 which is equivalent to 25cm
-    */
-    @serializable()
-    widthInMeters: number = .25;
-
-    /**
-     * The object moved around by the image. Make sure the size matches WidthInMeters.
-     */
-    @serializable(AssetReference)
-    object?: AssetReference;
-
-    /**
-     * If true, a new instance of the referenced object will be created for each tracked image. Enable this if you're re-using objects for multiple markers.
-     */
-    @serializable()
-    createObjectInstance: boolean = false;
-
-    /** Use this for static images (e.g. markers on the floor). Only the first few frames of new poses will be applied to the model. This will result in more stable tracking. 
-     * @default false
-    */
-    @serializable()
-    imageDoesNotMove: boolean = false;
-
-    /**
-     * Enable to hide the tracked object when the image is not tracked anymore. When disabled the tracked object will stay at the position it was last tracked at.
-     * @default true
-     */
-    @serializable()
-    hideWhenTrackingIsLost: boolean = true;
-}
-
-class ImageTrackingExtension implements IUSDExporterExtension {
-
-    get extensionName() { return "image-tracking"; }
-
-    private filename: string;
-    private widthInMeters: number;
-    private imageData: Uint8Array;
-
-    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
-        this.filename = filename;
-        this.imageData = imageData;
-        this.widthInMeters = widthInMeters;
-    }
-
-    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
-        const iOSVersion = DeviceUtilities.getiOSVersion();
-        const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
-        const workaroundForFB16119331 = majorVersion >= 18;
-        const multiplier = workaroundForFB16119331 ? 1 : 100;
-        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
-        writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
-        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
-        writer.closeBlock();
-    }
-
-    onBeforeBuildDocument(_context: USDZExporterContext) {
-        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
-        if (!imageTracking || !imageTracking.trackedImages) return;
-
-        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
-        if (imageTracking.trackedImages.length > 1) 
-        {
-            if (isDevEnvironment()) showBalloonWarning("USDZ: Only one tracked image is supported.");
-            console.warn("USDZ: Only one tracked image is supported.");
-        }
-    }
-
-    onAfterSerialize(context: USDZExporterContext) {
-        context.files['image_tracking/' + this.filename] = this.imageData;
-    }
-
-    onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, _context: USDZExporterContext) {
-        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
-        if (!imageTracking || !imageTracking.trackedImages) return;
-
-
-        for (const trackedImage of imageTracking.trackedImages) {
-            if (trackedImage.object?.asset === object) {
-                const exporter = GameObject.findObjectOfType(USDZExporter);
-                if (!exporter) continue;
-
-                const { scale, target } = exporter.getARScaleAndTarget();
-
-                // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
-                // On Android WebXR they're replaced by the tracked data
-                let parent = object;
-                const relativeMatrix = new Matrix4();
-                if (object !== target) {
-                    while (parent.parent && parent.parent !== target) {
-                        parent = parent.parent;
-                        relativeMatrix.premultiply(parent.matrix);
-                    }
-                }
-                const mat = relativeMatrix
-                    .clone()
-                    .invert()
-                // apply session root scale again after undoing the world transformation
-                model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
-
-                // Unfortunately looks like Apple's docs are incomplete:
-                // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
-                // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
-                // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
-                // model.extraSchemas.push("Preliminary_AnchoringAPI");
-                // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
-                    // writer.appendLine( `token preliminary:anchoring:type = "image"` );
-                    // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
-                // });
-
-                // We can only apply this to the first tracked image, more are not supported by QuickLook.
-                break;
-            }
-        }
-    }
-}
-
-/**
- * @category XR
- * @group Components
- */
-export class WebXRImageTracking extends Behaviour {
-
-    @serializable(WebXRImageTrackingModel)
-    trackedImages?: WebXRImageTrackingModel[];
-
-    /** Applies smoothing based on detected jitter to the tracked image. */
-    @serializable()
-    smooth: boolean = true;
-
-    private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
-    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
-
-    /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
-    get supported() { return this._supported; }
-
-    private _supported: boolean = true;
-
-    awake(): void {
-        if (debug) console.log(this)
-        if (!this.trackedImages) return;
-        for (const trackedImage of this.trackedImages) {
-            if (trackedImage.image) {
-                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
-                    // already loaded
-                }
-                else {
-                    const url = trackedImage.image;
-                    WebXRImageTracking._imageElements.set(url, null);
-                    const imageElement = document.createElement("img") as HTMLImageElement;
-                    imageElement.src = url;
-                    imageElement.addEventListener("load", async () => {
-                        const img = await createImageBitmap(imageElement);
-                        WebXRImageTracking._imageElements.set(url, img);
-
-                        // read back Uint8Array to use in USDZ - 
-                        // TODO better would be to do that once we actually need it
-                        const canvas = await imageToCanvas(img);
-                        if (canvas) {
-                            const blob = await canvas.convertToBlob({ type: 'image/png' });
-                            const arrayBuffer = await blob.arrayBuffer();
-
-                            const exporter = GameObject.findObjectOfType(USDZExporter);
-                            if (exporter && this.trackedImages) {
-                                exporter.extensions.push(
-                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
-                                );
-                                exporter.anchoringType = "image";
-                            }
-                        }
-                    });
-                }
-            }
-        }
-    }
-
-    onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array<any> }): void {
-        // console.log("onXRRequested", args, this.trackedImages)
-        if (this.trackedImages) {
-            args.optionalFeatures = args.optionalFeatures || [];
-            if (!args.optionalFeatures.includes("image-tracking"))
-                args.optionalFeatures.push("image-tracking");
-
-            args.trackedImages = [];
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
-                    const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
-                    if (bitmap) {
-                        this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
-                        args.trackedImages.push({
-                            image: bitmap,
-                            widthInMeters: trackedImage.widthInMeters
-                        });
-                    }
-                }
-            }
-        }
-    }
-
-    onEnterXR(_args: NeedleXREventArgs): void {
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    // capture the initial state of tracked images in the scene to restore them when the session ends
-                    const obj = trackedImage.object.asset as Object3D;
-                    if (!obj.userData) obj.userData = {};
-                    const state: InitialTrackedObjectState = {
-                        visible: obj.visible,
-                        parent: obj.parent,
-                        matrix: obj.matrix.clone()
-                    };
-                    obj.userData["image-tracking"] = state;
-                }
-            }
-        }
-        // clear out all frame counters for tracking
-        for (const trackedData of this.imageToObjectMap.values()) {
-            trackedData.frames = 0;
-        }
-    };
-
-    onLeaveXR(_args: NeedleXREventArgs): void {
-
-        if(!this.supported && DeviceUtilities.isAndroidDevice()) {
-            showBalloonWarning(this.webXRIncubationsWarning);
-        }
-
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    const obj = trackedImage.object.asset as Object3D;
-                    if (obj.userData) {
-                        // restore the initial state of tracked images in the scene
-                        const state = obj.userData["image-tracking"] as InitialTrackedObjectState | undefined;
-                        if (state) {
-                            obj.visible = state.visible;
-                            state.parent?.add(obj);
-                            obj.matrix.copy(state.matrix);
-                            obj.matrix.decompose(obj.position, obj.quaternion, obj.scale);
-                        }
-                        delete obj.userData["image-tracking"];
-                    }
-                }
-            }
-        }
-    }
-
-    private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: Object3D | null, frames: number, lastTrackingTime: number }>();
-    private readonly currentImages: WebXRTrackedImage[] = [];
-
-    
-    private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
-
-    onUpdateXR(args: NeedleXREventArgs): void {
-        this.currentImages.length = 0;
-
-        const frame = args.xr.frame;
-        if (!frame) return;
-
-        if (!("getImageTrackingResults" in frame)) {
-            if (!this["didPrintWarning"]) {
-                this["didPrintWarning"] = true;
-                console.log(this.webXRIncubationsWarning);
-            }
-            this._supported = false;
-            showBalloonWarning(this.webXRIncubationsWarning);
-            return;
-        }
-        // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch
-        // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled
-        else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) {
-            // Image tracking is not enabled for this session
-            return;
-        }
-        else if (frame.session && typeof frame.getImageTrackingResults === "function") {
-            const results = frame.getImageTrackingResults();
-            if (results.length > 0) {
-                const space = this.context.renderer.xr.getReferenceSpace();
-                if (space) {
-                    for (const result of results) {
-                        const state = result.trackingState;
-                        const imageIndex = result.index;
-                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
-                        if (trackedImage) {
-                            const pose = frame.getPose(result.imageSpace, space);
-                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
-                            this.currentImages.push(imageData);
-                        }
-                        else {
-                            if (debug) {
-                                console.warn("No tracked image for index", imageIndex);
-                            }
-                        }
-                    }
-                    if (this.currentImages.length > 0) {
-                        try {
-                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
-                            this.onImageTrackingUpdate(this.currentImages);
-                        }
-                        catch (e) {
-                            console.error(e);
-                        }
-                    }
-                }
-            }
-        }
-
-        // disable any objects that are no longer tracked
-        /** time in millis */
-        const hysteresis = 1000;
-        for (const [key, value] of this.imageToObjectMap) {
-            if (!value.object || !key) continue;
-            // If the user disallowed hiding the object when tracking is lost, skip this
-            if (key.hideWhenTrackingIsLost === false) continue;
-            let found = false;
-            for (const trackedImage of this.currentImages) {
-                if (trackedImage.model === key) {
-                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
-                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
-                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
-                        found = true;
-                        break;
-                    }
-                }
-            }
-            if (!found) {
-                GameObject.setActive(value.object, false);
-            }
-        }
-    }
-
-
-    private onImageTrackingUpdate = (images: WebXRTrackedImage[]) => {
-        const xr = NeedleXRSession.active;
-        if (!xr) return;
-
-
-        for (const image of images) {
-            const model = image.model;
-            const isTracked = image.state === "tracked";
-            // don't do anything if we don't have an object to track - can be handled externally through events
-            if (!model.object) continue;
-
-            let trackedData = this.imageToObjectMap.get(model);
-            if (trackedData === undefined) {
-                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
-                this.imageToObjectMap.set(model, trackedData);
-
-                model.object.loadAssetAsync().then((asset: Object3D | null) => {
-                    if (model.createObjectInstance && asset) {
-                        asset = GameObject.instantiate(asset);
-                    }
-
-                    if (asset) {
-                        trackedData!.object = asset;
-
-                        // workaround for instancing currently not properly updating 
-                        // instanced objects become visible when the image is recognized for the second time
-                        // we need to look into this further https://linear.app/needle/issue/NE-3936
-                        for (const rend of asset.getComponentsInChildren(Renderer)) {
-                            rend.setInstancingEnabled(false);
-                        }
-
-                        // make sure to parent to the WebXR.rig
-                        if (xr.rig) {
-                            xr.rig.gameObject.add(asset);
-                            image.applyToObject(asset);
-                            if (!(asset as IGameObject).activeSelf)
-                                GameObject.setActive(asset, true);
-                            // InstancingUtil.markDirty(asset);
-                        }
-                        else {
-                            console.warn("XRImageTracking: missing XRRig");
-                        }
-
-                    }
-                });
-            }
-            else {
-                trackedData.frames++;
-                if (isTracked)
-                    trackedData.lastTrackingTime = Date.now();
-
-                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
-                // to improve the tracking quality a bit.
-                if (model.imageDoesNotMove && trackedData.frames > 10)
-                    continue;
-
-                if (!trackedData.object) continue;
-
-                if (xr.rig) {
-
-                    xr.rig.gameObject.add(trackedData.object);
-
-                    image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined);
-                    if (!(trackedData.object as IGameObject).activeSelf) {
-                        GameObject.setActive(trackedData.object, true);
-                    }
-                    // InstancingUtil.markDirty(trackedData.object);
-                }
-            }
-        }
-    }
+import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
+import { Object3DEventMap } from "three";
+
+import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
+import { AssetReference } from "../../engine/engine_addressables.js";
+import { serializable } from "../../engine/engine_serialization.js";
+import { IGameObject } from "../../engine/engine_types.js";
+import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
+import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/api.js";
+import { IUSDExporterExtension } from "../../engine-components/export/usdz/Extension.js";
+import { imageToCanvas, USDObject, USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
+import { Behaviour, GameObject } from "../Component.js";
+import { Renderer } from "../Renderer.js";
+
+// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
+
+const debug = getParam("debugimagetracking");
+
+export class WebXRTrackedImage {
+
+    get url(): string { return this._trackedImage.image ?? ""; }
+    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
+    get bitmap(): ImageBitmap { return this._bitmap; }
+    get model(): WebXRImageTrackingModel { return this._trackedImage; }
+    readonly measuredSize: number;
+    readonly state: "tracked" | "emulated";
+
+    /** Copy the image position to a vector */
+    getPosition(vec: Vector3) {
+        this.ensureTransformData();
+        vec.copy(this._position);
+        return vec;
+    }
+
+    /** Copy the image rotation to a quaternion */
+    getQuaternion(quat: Quaternion) {
+        this.ensureTransformData();
+        quat.copy(this._rotation);
+        return quat;
+    }
+
+    applyToObject(object: Object3D, t01: number | undefined = undefined) {
+        this.ensureTransformData();
+        // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly
+        const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05;
+        if (t01) t01 *= Math.max(1, changeAmount);
+        if (t01 === undefined || t01 >= 1) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+            // InstancingUtil.markDirty(object);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+            // InstancingUtil.markDirty(object);
+        }
+    }
+
+    private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
+    private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
+    private _position!: Vector3;
+    private _rotation!: Quaternion;
+    private ensureTransformData() {
+        if (!this._position) {
+            this._position = WebXRTrackedImage._positionBuffer.get();
+            this._rotation = WebXRTrackedImage._rotationBuffer.get();
+            const t = this._pose.transform as XRRigidTransform;
+            const converted = NeedleXRSession.active!.convertSpace(t);
+            this._position.copy(converted?.position);
+            this._rotation.copy(converted?.quaternion);
+        }
+    }
+
+    private readonly _trackingComponent: WebXRImageTracking;
+    private readonly _trackedImage: WebXRImageTrackingModel;
+    private readonly _bitmap: ImageBitmap;
+    private readonly _pose: any;
+
+    constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
+        this._trackingComponent = context;;
+        this._trackedImage = trackedImage;
+        this._bitmap = bitmap;
+        this.measuredSize = measuredSize;
+        this.state = state;
+        this._pose = pose;
+    }
+
+}
+
+declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
+
+/** Initial state of assigned tracked image objects that are already present in the scene
+ * This is used to restore the state of the object when the webxr session has ended to return to the original state
+ */
+declare type InitialTrackedObjectState = {
+    visible: boolean;
+    parent: Object3D | undefined | null;
+    matrix: Matrix4;
+}
+/**
+ * WebXRImageTracking allows you to track images in the real world and place objects on top of them.  
+ * This component is only available in WebXR sessions.  
+ * The WebXRImageTrackingModel contains the image to track, the object to place on top of the image, and the size of the image as well as settings for the tracking.  
+ * Used by the {@link WebXRImageTracking} component
+ */
+export class WebXRImageTrackingModel {
+
+    /**
+     * Tracked image marker url. Make sure the image has good contrast and unique features to improve the tracking quality.
+     */
+    @serializable(URL)
+    image?: string;
+
+    /** Make sure this matches your physical marker size! Otherwise the tracked object will \"swim\" above or below the marker. 
+     * @default 0.25 which is equivalent to 25cm
+    */
+    @serializable()
+    widthInMeters: number = .25;
+
+    /**
+     * The object moved around by the image. Make sure the size matches WidthInMeters.
+     */
+    @serializable(AssetReference)
+    object?: AssetReference;
+
+    /**
+     * If true, a new instance of the referenced object will be created for each tracked image. Enable this if you're re-using objects for multiple markers.
+     */
+    @serializable()
+    createObjectInstance: boolean = false;
+
+    /** Use this for static images (e.g. markers on the floor). Only the first few frames of new poses will be applied to the model. This will result in more stable tracking. 
+     * @default false
+    */
+    @serializable()
+    imageDoesNotMove: boolean = false;
+
+    /**
+     * Enable to hide the tracked object when the image is not tracked anymore. When disabled the tracked object will stay at the position it was last tracked at.
+     * @default true
+     */
+    @serializable()
+    hideWhenTrackingIsLost: boolean = true;
+}
+
+class ImageTrackingExtension implements IUSDExporterExtension {
+
+    get extensionName() { return "image-tracking"; }
+
+    private filename: string;
+    private widthInMeters: number;
+    private imageData: Uint8Array;
+
+    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
+        this.filename = filename;
+        this.imageData = imageData;
+        this.widthInMeters = widthInMeters;
+    }
+
+    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
+        const iOSVersion = DeviceUtilities.getiOSVersion();
+        const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
+        const workaroundForFB16119331 = majorVersion >= 18;
+        const multiplier = workaroundForFB16119331 ? 1 : 100;
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * multiplier).toFixed(8));
+        writer.closeBlock();
+    }
+
+    onBeforeBuildDocument(_context: USDZExporterContext) {
+        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
+        if (!imageTracking || !imageTracking.trackedImages) return;
+
+        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
+        if (imageTracking.trackedImages.length > 1) 
+        {
+            if (isDevEnvironment()) showBalloonWarning("USDZ: Only one tracked image is supported.");
+            console.warn("USDZ: Only one tracked image is supported.");
+        }
+    }
+
+    onAfterSerialize(context: USDZExporterContext) {
+        context.files['image_tracking/' + this.filename] = this.imageData;
+    }
+
+    onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, _context: USDZExporterContext) {
+        const imageTracking = GameObject.findObjectOfType(WebXRImageTracking);
+        if (!imageTracking || !imageTracking.trackedImages) return;
+
+
+        for (const trackedImage of imageTracking.trackedImages) {
+            if (trackedImage.object?.asset === object) {
+                const exporter = GameObject.findObjectOfType(USDZExporter);
+                if (!exporter) continue;
+
+                const { scale, target } = exporter.getARScaleAndTarget();
+
+                // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
+                // On Android WebXR they're replaced by the tracked data
+                let parent = object;
+                const relativeMatrix = new Matrix4();
+                if (object !== target) {
+                    while (parent.parent && parent.parent !== target) {
+                        parent = parent.parent;
+                        relativeMatrix.premultiply(parent.matrix);
+                    }
+                }
+                const mat = relativeMatrix
+                    .clone()
+                    .invert()
+                // apply session root scale again after undoing the world transformation
+                model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
+
+                // Unfortunately looks like Apple's docs are incomplete:
+                // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
+                // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
+                // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
+                // model.extraSchemas.push("Preliminary_AnchoringAPI");
+                // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
+                    // writer.appendLine( `token preliminary:anchoring:type = "image"` );
+                    // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
+                // });
+
+                // We can only apply this to the first tracked image, more are not supported by QuickLook.
+                break;
+            }
+        }
+    }
+}
+
+/**
+ * @category XR
+ * @group Components
+ */
+export class WebXRImageTracking extends Behaviour {
+
+    @serializable(WebXRImageTrackingModel)
+    trackedImages?: WebXRImageTrackingModel[];
+
+    /** Applies smoothing based on detected jitter to the tracked image. */
+    @serializable()
+    smooth: boolean = true;
+
+    private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
+    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
+
+    /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
+    get supported() { return this._supported; }
+
+    private _supported: boolean = true;
+
+    awake(): void {
+        if (debug) console.log(this)
+        if (!this.trackedImages) return;
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image) {
+                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
+                    // already loaded
+                }
+                else {
+                    const url = trackedImage.image;
+                    WebXRImageTracking._imageElements.set(url, null);
+                    const imageElement = document.createElement("img") as HTMLImageElement;
+                    imageElement.src = url;
+                    imageElement.addEventListener("load", async () => {
+                        const img = await createImageBitmap(imageElement);
+                        WebXRImageTracking._imageElements.set(url, img);
+
+                        // read back Uint8Array to use in USDZ - 
+                        // TODO better would be to do that once we actually need it
+                        const canvas = await imageToCanvas(img);
+                        if (canvas) {
+                            const blob = await canvas.convertToBlob({ type: 'image/png' });
+                            const arrayBuffer = await blob.arrayBuffer();
+
+                            const exporter = GameObject.findObjectOfType(USDZExporter);
+                            if (exporter && this.trackedImages) {
+                                exporter.extensions.push(
+                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters)
+                                );
+                                exporter.anchoringType = "image";
+                            }
+                        }
+                    });
+                }
+            }
+        }
+    }
+
+    onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array<any> }): void {
+        // console.log("onXRRequested", args, this.trackedImages)
+        if (this.trackedImages) {
+            args.optionalFeatures = args.optionalFeatures || [];
+            if (!args.optionalFeatures.includes("image-tracking"))
+                args.optionalFeatures.push("image-tracking");
+
+            args.trackedImages = [];
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
+                    const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
+                    if (bitmap) {
+                        this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
+                        args.trackedImages.push({
+                            image: bitmap,
+                            widthInMeters: trackedImage.widthInMeters
+                        });
+                    }
+                }
+            }
+        }
+    }
+
+    onEnterXR(_args: NeedleXREventArgs): void {
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    // capture the initial state of tracked images in the scene to restore them when the session ends
+                    const obj = trackedImage.object.asset as Object3D;
+                    if (!obj.userData) obj.userData = {};
+                    const state: InitialTrackedObjectState = {
+                        visible: obj.visible,
+                        parent: obj.parent,
+                        matrix: obj.matrix.clone()
+                    };
+                    obj.userData["image-tracking"] = state;
+                }
+            }
+        }
+        // clear out all frame counters for tracking
+        for (const trackedData of this.imageToObjectMap.values()) {
+            trackedData.frames = 0;
+        }
+    };
+
+    onLeaveXR(_args: NeedleXREventArgs): void {
+
+        if(!this.supported && DeviceUtilities.isAndroidDevice()) {
+            showBalloonWarning(this.webXRIncubationsWarning);
+        }
+
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    const obj = trackedImage.object.asset as Object3D;
+                    if (obj.userData) {
+                        // restore the initial state of tracked images in the scene
+                        const state = obj.userData["image-tracking"] as InitialTrackedObjectState | undefined;
+                        if (state) {
+                            obj.visible = state.visible;
+                            state.parent?.add(obj);
+                            obj.matrix.copy(state.matrix);
+                            obj.matrix.decompose(obj.position, obj.quaternion, obj.scale);
+                        }
+                        delete obj.userData["image-tracking"];
+                    }
+                }
+            }
+        }
+    }
+
+    private readonly imageToObjectMap = new Map<WebXRImageTrackingModel, { object: Object3D | null, frames: number, lastTrackingTime: number }>();
+    private readonly currentImages: WebXRTrackedImage[] = [];
+
+    
+    private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
+
+    onUpdateXR(args: NeedleXREventArgs): void {
+        this.currentImages.length = 0;
+
+        const frame = args.xr.frame;
+        if (!frame) return;
+
+        if (!("getImageTrackingResults" in frame)) {
+            if (!this["didPrintWarning"]) {
+                this["didPrintWarning"] = true;
+                console.log(this.webXRIncubationsWarning);
+            }
+            this._supported = false;
+            showBalloonWarning(this.webXRIncubationsWarning);
+            return;
+        }
+        // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch
+        // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled
+        else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) {
+            // Image tracking is not enabled for this session
+            return;
+        }
+        else if (frame.session && typeof frame.getImageTrackingResults === "function") {
+            const results = frame.getImageTrackingResults();
+            if (results.length > 0) {
+                const space = this.context.renderer.xr.getReferenceSpace();
+                if (space) {
+                    for (const result of results) {
+                        const state = result.trackingState;
+                        const imageIndex = result.index;
+                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
+                        if (trackedImage) {
+                            const pose = frame.getPose(result.imageSpace, space);
+                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
+                            this.currentImages.push(imageData);
+                        }
+                        else {
+                            if (debug) {
+                                console.warn("No tracked image for index", imageIndex);
+                            }
+                        }
+                    }
+                    if (this.currentImages.length > 0) {
+                        try {
+                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
+                            this.onImageTrackingUpdate(this.currentImages);
+                        }
+                        catch (e) {
+                            console.error(e);
+                        }
+                    }
+                }
+            }
+        }
+
+        // disable any objects that are no longer tracked
+        /** time in millis */
+        const hysteresis = 1000;
+        for (const [key, value] of this.imageToObjectMap) {
+            if (!value.object || !key) continue;
+            // If the user disallowed hiding the object when tracking is lost, skip this
+            if (key.hideWhenTrackingIsLost === false) continue;
+            let found = false;
+            for (const trackedImage of this.currentImages) {
+                if (trackedImage.model === key) {
+                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
+                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
+                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            if (!found) {
+                GameObject.setActive(value.object, false);
+            }
+        }
+    }
+
+
+    private onImageTrackingUpdate = (images: WebXRTrackedImage[]) => {
+        const xr = NeedleXRSession.active;
+        if (!xr) return;
+
+
+        for (const image of images) {
+            const model = image.model;
+            const isTracked = image.state === "tracked";
+            // don't do anything if we don't have an object to track - can be handled externally through events
+            if (!model.object) continue;
+
+            let trackedData = this.imageToObjectMap.get(model);
+            if (trackedData === undefined) {
+                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
+                this.imageToObjectMap.set(model, trackedData);
+
+                model.object.loadAssetAsync().then((asset: Object3D | null) => {
+                    if (model.createObjectInstance && asset) {
+                        asset = GameObject.instantiate(asset);
+                    }
+
+                    if (asset) {
+                        trackedData!.object = asset;
+
+                        // workaround for instancing currently not properly updating 
+                        // instanced objects become visible when the image is recognized for the second time
+                        // we need to look into this further https://linear.app/needle/issue/NE-3936
+                        for (const rend of asset.getComponentsInChildren(Renderer)) {
+                            rend.setInstancingEnabled(false);
+                        }
+
+                        // make sure to parent to the WebXR.rig
+                        if (xr.rig) {
+                            xr.rig.gameObject.add(asset);
+                            image.applyToObject(asset);
+                            if (!(asset as IGameObject).activeSelf)
+                                GameObject.setActive(asset, true);
+                            // InstancingUtil.markDirty(asset);
+                        }
+                        else {
+                            console.warn("XRImageTracking: missing XRRig");
+                        }
+
+                    }
+                });
+            }
+            else {
+                trackedData.frames++;
+                if (isTracked)
+                    trackedData.lastTrackingTime = Date.now();
+
+                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
+                // to improve the tracking quality a bit.
+                if (model.imageDoesNotMove && trackedData.frames > 10)
+                    continue;
+
+                if (!trackedData.object) continue;
+
+                if (xr.rig) {
+
+                    xr.rig.gameObject.add(trackedData.object);
+
+                    image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined);
+                    if (!(trackedData.object as IGameObject).activeSelf) {
+                        GameObject.setActive(trackedData.object, true);
+                    }
+                    // InstancingUtil.markDirty(trackedData.object);
+                }
+            }
+        }
+    }
 }