@needle-tools/engine 4.11.5-next.f5ee735 → 4.11.6

package/lib/engine-components/webxr/WebXRImageTracking.js

@@ -1,591 +1,591 @@
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
-import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
-import { AssetReference } from "../../engine/engine_addressables.js";
-import { serializable } from "../../engine/engine_serialization.js";
-import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
-import { NeedleXRSession } from "../../engine/xr/api.js";
-import { imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
-import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
-import { Behaviour, GameObject } from "../Component.js";
-import { Renderer } from "../Renderer.js";
-// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
-const debug = getParam("debugimagetracking");
-export class WebXRTrackedImage {
-    get url() { return this._trackedImage.image ?? ""; }
-    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
-    get bitmap() { return this._bitmap; }
-    get model() { return this._trackedImage; }
-    measuredSize;
-    state;
-    /** Copy the image position to a vector */
-    getPosition(vec) {
-        this.ensureTransformData();
-        vec.copy(this._position);
-        return vec;
-    }
-    /** Copy the image rotation to a quaternion */
-    getQuaternion(quat) {
-        this.ensureTransformData();
-        quat.copy(this._rotation);
-        return quat;
-    }
-    applyToObject(object, t01 = undefined) {
-        this.ensureTransformData();
-        // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly
-        const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05;
-        if (t01)
-            t01 *= Math.max(1, changeAmount);
-        if (t01 === undefined || t01 >= 1) {
-            object.position.copy(this._position);
-            object.quaternion.copy(this._rotation);
-            // InstancingUtil.markDirty(object);
-        }
-        else {
-            t01 = Math.max(0, Math.min(1, t01));
-            object.position.lerp(this._position, t01);
-            object.quaternion.slerp(this._rotation, t01);
-            // InstancingUtil.markDirty(object);
-        }
-    }
-    static _positionBuffer = new CircularBuffer(() => new Vector3(), 20);
-    static _rotationBuffer = new CircularBuffer(() => new Quaternion(), 20);
-    _position;
-    _rotation;
-    ensureTransformData() {
-        if (!this._position) {
-            this._position = WebXRTrackedImage._positionBuffer.get();
-            this._rotation = WebXRTrackedImage._rotationBuffer.get();
-            const t = this._pose.transform;
-            const converted = NeedleXRSession.active.convertSpace(t);
-            this._position.copy(converted?.position);
-            this._rotation.copy(converted?.quaternion);
-        }
-    }
-    _trackingComponent;
-    _trackedImage;
-    _bitmap;
-    _pose;
-    constructor(context, trackedImage, bitmap, measuredSize, state, pose) {
-        this._trackingComponent = context;
-        ;
-        this._trackedImage = trackedImage;
-        this._bitmap = bitmap;
-        this.measuredSize = measuredSize;
-        this.state = state;
-        this._pose = pose;
-    }
-}
-/**
- * WebXRImageTracking allows you to track images in the real world and place objects on top of them.
- * This component is only available in WebXR sessions.
- * The WebXRImageTrackingModel contains the image to track, the object to place on top of the image, and the size of the image as well as settings for the tracking.
- * Used by the {@link WebXRImageTracking} component
- */
-export class WebXRImageTrackingModel {
-    constructor(params) {
-        this.image = params.url;
-        this.widthInMeters = params.widthInMeters;
-        if (params.object instanceof Object3D) {
-            this.object = new AssetReference({ asset: params.object });
-        }
-        else
-            this.object = params.object;
-        if (params.createObjectInstance !== undefined)
-            this.createObjectInstance = params.createObjectInstance;
-        if (params.imageDoesNotMove !== undefined)
-            this.imageDoesNotMove = params.imageDoesNotMove;
-        if (params.hideWhenTrackingIsLost !== undefined)
-            this.hideWhenTrackingIsLost = params.hideWhenTrackingIsLost;
-    }
-    /**
-     * Tracked image marker url. Make sure the image has good contrast and unique features to improve the tracking quality.
-     */
-    image;
-    /** Make sure this matches your physical marker size! Otherwise the tracked object will \"swim\" above or below the marker.
-     * @default 0.25 which is equivalent to 25cm
-    */
-    widthInMeters = .25;
-    /**
-     * The object moved around by the image. Make sure the size matches WidthInMeters.
-     */
-    object;
-    /**
-     * If true, a new instance of the referenced object will be created for each tracked image. Enable this if you're re-using objects for multiple markers.
-     */
-    createObjectInstance = false;
-    /** Use this for static images (e.g. markers on the floor). Only the first few frames of new poses will be applied to the model. This will result in more stable tracking.
-     * @default false
-    */
-    imageDoesNotMove = false;
-    /**
-     * Enable to hide the tracked object when the image is not tracked anymore. When disabled the tracked object will stay at the position it was last tracked at.
-     * @default true
-     */
-    hideWhenTrackingIsLost = true;
-    /**
-     * Get the name of the image file from the url
-     */
-    getNameFromUrl() {
-        if (this.image) {
-            const parts = this.image.split("/");
-            return parts[parts.length - 1];
-        }
-        return null;
-    }
-}
-__decorate([
-    serializable(URL)
-], WebXRImageTrackingModel.prototype, "image", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "widthInMeters", void 0);
-__decorate([
-    serializable(AssetReference)
-], WebXRImageTrackingModel.prototype, "object", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "createObjectInstance", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "imageDoesNotMove", void 0);
-__decorate([
-    serializable()
-], WebXRImageTrackingModel.prototype, "hideWhenTrackingIsLost", void 0);
-class ImageTrackingExtension {
-    exporter;
-    component;
-    isImageTrackingExtension = true;
-    get extensionName() { return "image-tracking"; }
-    constructor(exporter, component) {
-        this.exporter = exporter;
-        this.component = component;
-        if (debug)
-            console.log(this);
-        this.exporter.anchoringType = "image";
-    }
-    // set during export
-    shouldExport = true;
-    filename = null;
-    imageModel = null;
-    onBeforeBuildDocument(_context) {
-        // check if this extension is the first image tracking extension in the list
-        // since iOS can only track one image at a time we only allow one image tracking extension to be active
-        // we have to determine this at the earlierst export callback
-        // all subsequent export callbacks should then check is shouldExport is set to true
-        // this should only be the case for exactly one extension
-        const index = this.exporter.extensions
-            .filter(e => {
-            const ext = e;
-            return ext.isImageTrackingExtension && ext.component.activeAndEnabled && ext.component.trackedImages?.length > 0;
-        })
-            .indexOf(this);
-        this.shouldExport = index === 0;
-        if (!this.shouldExport)
-            return;
-        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
-        if (this.component.trackedImages?.length > 1) {
-            if (debug || isDevEnvironment()) {
-                showBalloonWarning("USDZ: Only one tracked image is supported.");
-                console.warn("USDZ: Only one tracked image is supported. Will choose the first one in the trackedImages list");
-            }
-        }
-    }
-    onAfterHierarchy(_context, writer) {
-        if (!this.shouldExport)
-            return;
-        const iOSVersion = DeviceUtilities.getiOSVersion();
-        const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
-        const workaroundForFB16119331 = majorVersion >= 18;
-        const multiplier = workaroundForFB16119331 ? 1 : 100;
-        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
-        writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
-        writer.appendLine(`uniform double physicalWidth = ` + (this.imageModel.widthInMeters * multiplier).toFixed(8));
-        writer.closeBlock();
-    }
-    async onAfterSerialize(context) {
-        if (!this.shouldExport)
-            return;
-        const imageModel = this.imageModel;
-        const img = _imageElements.get(imageModel.image);
-        const canvas = await imageToCanvas(img);
-        const blob = await canvas.convertToBlob({ type: 'image/png' });
-        const arrayBuffer = await blob.arrayBuffer();
-        context.files['image_tracking/' + this.filename] = new Uint8Array(arrayBuffer);
-    }
-    onExportObject(object, model, _context) {
-        if (!this.shouldExport)
-            return;
-        const imageTracking = this.component;
-        if (!imageTracking || !imageTracking.trackedImages?.length || !imageTracking.activeAndEnabled)
-            return;
-        // we only care about the first image
-        // We can only apply this to the first tracked image, more are not supported by QuickLook.
-        const trackedImage = imageTracking.trackedImages[0];
-        if (trackedImage.object?.asset === object) {
-            this.imageModel = trackedImage;
-            this.filename = trackedImage.getNameFromUrl() || "marker.png";
-            const { scale, target } = this.exporter.getARScaleAndTarget();
-            // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
-            // On Android WebXR they're replaced by the tracked data
-            let parent = object;
-            const relativeMatrix = new Matrix4();
-            if (object !== target) {
-                while (parent && parent.parent && parent.parent !== target) {
-                    parent = parent.parent;
-                    relativeMatrix.premultiply(parent.matrix);
-                }
-            }
-            const mat = relativeMatrix
-                .clone()
-                .invert();
-            // apply session root scale again after undoing the world transformation
-            model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
-            // Unfortunately looks like Apple's docs are incomplete:
-            // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
-            // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
-            // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
-            // model.extraSchemas.push("Preliminary_AnchoringAPI");
-            // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
-            // writer.appendLine( `token preliminary:anchoring:type = "image"` );
-            // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
-            // });
-        }
-    }
-}
-/**
- * Add this component to a object to enable image tracking in WebXR sessions.
- *
- * You need to add at least one {@link WebXRImageTrackingModel} to the `trackedImages` array to define which images to track and which objects to place on top of them.
- *
- * **NOTE:** For Android devices, image tracking currently requires the user to enable the `chrome://flags/#webxr-incubations` flag in Chrome.
- *
- * **NOTE:** For iOS only one image can be tracked at a time. If you have multiple images in the `trackedImages` array, only the first one will be tracked. You can use the {@link setPrimaryImage} method to change which image is tracked before entering the XR session.
- *
- * - Example: https://samples.needle.tools/image-tracking
- *
- * @category XR
- * @group Components
- */
-export class WebXRImageTracking extends Behaviour {
-    /**
-     * If you have multiple images in your application to track then this method is useful for iOS AR where only the first image can be tracked.
-     * Call this method will set the given image as the first one in the array so it will be used for tracking.
-     */
-    setPrimaryImage(image) {
-        const index = this.trackedImages.indexOf(image);
-        if (index >= 0) {
-            const current = this.trackedImages[0];
-            if (current !== image) {
-                this.trackedImages[0] = image;
-                this.trackedImages[index] = current;
-            }
-        }
-        else
-            console.warn(`[WebXRImageTracking] Can not set primary: image not found in 'trackedImages' array ${image.image}`);
-    }
-    /**
-     * Add an image to track. If the image is already in the trackedImages array it won't be added again.
-     * Note: that adding images at runtime *while* in AR is not supported.
-     *
-     * @example
-     * ```ts
-     * const imageTracking = GameObject.getComponent(myGameObject, WebXRImageTracking);
-     * const image = new WebXRImageTrackingModel({ url: "https://example.com/my-marker.png", widthInMeters: 0.2, object: myObject });
-     * imageTracking.addImage(image, true); // add and set as primary image
-     * ```
-     */
-    addImage(image, asPrimary = false) {
-        if (!this.trackedImages.includes(image)) {
-            this.trackedImages.push(image);
-            loadImage(image.image);
-        }
-        if (asPrimary)
-            this.setPrimaryImage(image);
-    }
-    /**
-     * List of images to track in the WebXR session. Use {@link WebXRImageTrackingModel} to define each image and the object to place on top of it.
-     *
-     * Use the `addImage()` and `setPrimaryImage()` methods to modify this array at runtime.
-     */
-    trackedImages = [];
-    /** Applies smoothing based on detected jitter to the tracked image. */
-    smooth = true;
-    trackedImageIndexMap = new Map();
-    /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
-    get supported() { return this._supported; }
-    _supported = true;
-    /** @internal */
-    awake() {
-        if (debug)
-            console.log(this);
-        if (!this.trackedImages)
-            return;
-        for (const trackedImage of this.trackedImages) {
-            if (trackedImage.image) {
-                loadImage(trackedImage.image);
-            }
-        }
-    }
-    /** @internal */
-    onEnable() {
-        USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport);
-    }
-    /** @internal */
-    onDisable() {
-        USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport);
-    }
-    onBeforeUSDZExport = (args) => {
-        if (this.activeAndEnabled && this.trackedImages?.length) {
-            args.exporter.extensions.push(new ImageTrackingExtension(args.exporter, this));
-        }
-    };
-    /** @internal */
-    onBeforeXR(_mode, args) {
-        // console.log("onXRRequested", args, this.trackedImages)
-        if (this.trackedImages) {
-            args.optionalFeatures = args.optionalFeatures || [];
-            if (!args.optionalFeatures.includes("image-tracking"))
-                args.optionalFeatures.push("image-tracking");
-            args.trackedImages = [];
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
-                    const bitmap = _imageElements.get(trackedImage.image);
-                    if (bitmap) {
-                        this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
-                        args.trackedImages.push({
-                            image: bitmap,
-                            widthInMeters: trackedImage.widthInMeters
-                        });
-                    }
-                }
-            }
-        }
-    }
-    /** @internal */
-    onEnterXR(_args) {
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    // capture the initial state of tracked images in the scene to restore them when the session ends
-                    const obj = trackedImage.object.asset;
-                    if (!obj.userData)
-                        obj.userData = {};
-                    const state = {
-                        visible: obj.visible,
-                        parent: obj.parent,
-                        matrix: obj.matrix.clone()
-                    };
-                    obj.userData["image-tracking"] = state;
-                }
-            }
-        }
-        // clear out all frame counters for tracking
-        for (const trackedData of this.imageToObjectMap.values()) {
-            trackedData.frames = 0;
-        }
-    }
-    ;
-    /** @internal */
-    onLeaveXR(_args) {
-        if (!this.supported && DeviceUtilities.isAndroidDevice()) {
-            showBalloonWarning(this.webXRIncubationsWarning);
-        }
-        if (this.trackedImages) {
-            for (const trackedImage of this.trackedImages) {
-                if (trackedImage.object?.asset) {
-                    const obj = trackedImage.object.asset;
-                    if (obj.userData) {
-                        // restore the initial state of tracked images in the scene
-                        const state = obj.userData["image-tracking"];
-                        if (state) {
-                            obj.visible = state.visible;
-                            state.parent?.add(obj);
-                            obj.matrix.copy(state.matrix);
-                            obj.matrix.decompose(obj.position, obj.quaternion, obj.scale);
-                        }
-                        delete obj.userData["image-tracking"];
-                    }
-                }
-            }
-        }
-    }
-    imageToObjectMap = new Map();
-    currentImages = [];
-    webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
-    /** @internal */
-    onUpdateXR(args) {
-        this.currentImages.length = 0;
-        const frame = args.xr.frame;
-        if (!frame)
-            return;
-        if (!("getImageTrackingResults" in frame)) {
-            if (!this["didPrintWarning"]) {
-                this["didPrintWarning"] = true;
-                console.log(this.webXRIncubationsWarning);
-            }
-            this._supported = false;
-            showBalloonWarning(this.webXRIncubationsWarning);
-            return;
-        }
-        // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch
-        // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled
-        else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) {
-            // Image tracking is not enabled for this session
-            return;
-        }
-        else if (frame.session && typeof frame.getImageTrackingResults === "function") {
-            const results = frame.getImageTrackingResults();
-            if (results.length > 0) {
-                const space = this.context.renderer.xr.getReferenceSpace();
-                if (space) {
-                    for (const result of results) {
-                        const state = result.trackingState;
-                        const imageIndex = result.index;
-                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
-                        if (trackedImage) {
-                            const pose = frame.getPose(result.imageSpace, space);
-                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
-                            this.currentImages.push(imageData);
-                        }
-                        else {
-                            if (debug) {
-                                console.warn("No tracked image for index", imageIndex);
-                            }
-                        }
-                    }
-                    if (this.currentImages.length > 0) {
-                        try {
-                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
-                            this.onImageTrackingUpdate(this.currentImages);
-                        }
-                        catch (e) {
-                            console.error(e);
-                        }
-                    }
-                }
-            }
-        }
-        // disable any objects that are no longer tracked
-        /** time in millis */
-        const hysteresis = 1000;
-        for (const [key, value] of this.imageToObjectMap) {
-            if (!value.object || !key)
-                continue;
-            // If the user disallowed hiding the object when tracking is lost, skip this
-            if (key.hideWhenTrackingIsLost === false)
-                continue;
-            let found = false;
-            for (const trackedImage of this.currentImages) {
-                if (trackedImage.model === key) {
-                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
-                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
-                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
-                        found = true;
-                        break;
-                    }
-                }
-            }
-            if (!found) {
-                GameObject.setActive(value.object, false);
-            }
-        }
-    }
-    onImageTrackingUpdate = (images) => {
-        const xr = NeedleXRSession.active;
-        if (!xr)
-            return;
-        for (const image of images) {
-            const model = image.model;
-            const isTracked = image.state === "tracked";
-            // don't do anything if we don't have an object to track - can be handled externally through events
-            if (!model.object)
-                continue;
-            let trackedData = this.imageToObjectMap.get(model);
-            if (trackedData === undefined) {
-                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
-                this.imageToObjectMap.set(model, trackedData);
-                model.object.loadAssetAsync().then((asset) => {
-                    if (model.createObjectInstance && asset) {
-                        asset = GameObject.instantiate(asset);
-                    }
-                    if (asset) {
-                        trackedData.object = asset;
-                        // workaround for instancing currently not properly updating 
-                        // instanced objects become visible when the image is recognized for the second time
-                        // we need to look into this further https://linear.app/needle/issue/NE-3936
-                        for (const rend of asset.getComponentsInChildren(Renderer)) {
-                            rend.setInstancingEnabled(false);
-                        }
-                        // make sure to parent to the WebXR.rig
-                        if (xr.rig) {
-                            xr.rig.gameObject.add(asset);
-                            image.applyToObject(asset);
-                            if (!asset.activeSelf)
-                                GameObject.setActive(asset, true);
-                            // InstancingUtil.markDirty(asset);
-                        }
-                        else {
-                            console.warn("XRImageTracking: missing XRRig");
-                        }
-                    }
-                });
-            }
-            else {
-                trackedData.frames++;
-                if (isTracked)
-                    trackedData.lastTrackingTime = Date.now();
-                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
-                // to improve the tracking quality a bit.
-                if (model.imageDoesNotMove && trackedData.frames > 10)
-                    continue;
-                if (!trackedData.object)
-                    continue;
-                if (xr.rig) {
-                    xr.rig.gameObject.add(trackedData.object);
-                    image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined);
-                    if (!trackedData.object.activeSelf) {
-                        GameObject.setActive(trackedData.object, true);
-                    }
-                    // InstancingUtil.markDirty(trackedData.object);
-                }
-            }
-        }
-    };
-}
-__decorate([
-    serializable(WebXRImageTrackingModel)
-], WebXRImageTracking.prototype, "trackedImages", void 0);
-__decorate([
-    serializable()
-], WebXRImageTracking.prototype, "smooth", void 0);
-const _imageElements = new Map();
-const _imageLoadingPromises = new Map();
-async function loadImage(url) {
-    if (_imageElements.has(url)) {
-        if (_imageLoadingPromises.has(url))
-            return _imageLoadingPromises.get(url);
-        return Promise.resolve(true);
-    }
-    const promise = new Promise(res => {
-        _imageElements.set(url, null);
-        const imageElement = document.createElement("img");
-        imageElement.src = url;
-        imageElement.addEventListener("load", async () => {
-            const img = await createImageBitmap(imageElement);
-            _imageElements.set(url, img);
-            res(true);
-        });
-    });
-    _imageLoadingPromises.set(url, promise);
-    promise.finally(() => {
-        _imageLoadingPromises.delete(url);
-    });
-    return promise;
-}
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { Matrix4, Object3D, Quaternion, Vector3 } from "three";
+import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js";
+import { AssetReference } from "../../engine/engine_addressables.js";
+import { serializable } from "../../engine/engine_serialization.js";
+import { CircularBuffer, DeviceUtilities, getParam } from "../../engine/engine_utils.js";
+import { NeedleXRSession } from "../../engine/xr/api.js";
+import { imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter.js";
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js";
+import { Behaviour, GameObject } from "../Component.js";
+import { Renderer } from "../Renderer.js";
+// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
+const debug = getParam("debugimagetracking");
+export class WebXRTrackedImage {
+    get url() { return this._trackedImage.image ?? ""; }
+    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
+    get bitmap() { return this._bitmap; }
+    get model() { return this._trackedImage; }
+    measuredSize;
+    state;
+    /** Copy the image position to a vector */
+    getPosition(vec) {
+        this.ensureTransformData();
+        vec.copy(this._position);
+        return vec;
+    }
+    /** Copy the image rotation to a quaternion */
+    getQuaternion(quat) {
+        this.ensureTransformData();
+        quat.copy(this._rotation);
+        return quat;
+    }
+    applyToObject(object, t01 = undefined) {
+        this.ensureTransformData();
+        // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly
+        const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05;
+        if (t01)
+            t01 *= Math.max(1, changeAmount);
+        if (t01 === undefined || t01 >= 1) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+            // InstancingUtil.markDirty(object);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+            // InstancingUtil.markDirty(object);
+        }
+    }
+    static _positionBuffer = new CircularBuffer(() => new Vector3(), 20);
+    static _rotationBuffer = new CircularBuffer(() => new Quaternion(), 20);
+    _position;
+    _rotation;
+    ensureTransformData() {
+        if (!this._position) {
+            this._position = WebXRTrackedImage._positionBuffer.get();
+            this._rotation = WebXRTrackedImage._rotationBuffer.get();
+            const t = this._pose.transform;
+            const converted = NeedleXRSession.active.convertSpace(t);
+            this._position.copy(converted?.position);
+            this._rotation.copy(converted?.quaternion);
+        }
+    }
+    _trackingComponent;
+    _trackedImage;
+    _bitmap;
+    _pose;
+    constructor(context, trackedImage, bitmap, measuredSize, state, pose) {
+        this._trackingComponent = context;
+        ;
+        this._trackedImage = trackedImage;
+        this._bitmap = bitmap;
+        this.measuredSize = measuredSize;
+        this.state = state;
+        this._pose = pose;
+    }
+}
+/**
+ * WebXRImageTracking allows you to track images in the real world and place objects on top of them.
+ * This component is only available in WebXR sessions.
+ * The WebXRImageTrackingModel contains the image to track, the object to place on top of the image, and the size of the image as well as settings for the tracking.
+ * Used by the {@link WebXRImageTracking} component
+ */
+export class WebXRImageTrackingModel {
+    constructor(params) {
+        this.image = params.url;
+        this.widthInMeters = params.widthInMeters;
+        if (params.object instanceof Object3D) {
+            this.object = new AssetReference({ asset: params.object });
+        }
+        else
+            this.object = params.object;
+        if (params.createObjectInstance !== undefined)
+            this.createObjectInstance = params.createObjectInstance;
+        if (params.imageDoesNotMove !== undefined)
+            this.imageDoesNotMove = params.imageDoesNotMove;
+        if (params.hideWhenTrackingIsLost !== undefined)
+            this.hideWhenTrackingIsLost = params.hideWhenTrackingIsLost;
+    }
+    /**
+     * Tracked image marker url. Make sure the image has good contrast and unique features to improve the tracking quality.
+     */
+    image;
+    /** Make sure this matches your physical marker size! Otherwise the tracked object will \"swim\" above or below the marker.
+     * @default 0.25 which is equivalent to 25cm
+    */
+    widthInMeters = .25;
+    /**
+     * The object moved around by the image. Make sure the size matches WidthInMeters.
+     */
+    object;
+    /**
+     * If true, a new instance of the referenced object will be created for each tracked image. Enable this if you're re-using objects for multiple markers.
+     */
+    createObjectInstance = false;
+    /** Use this for static images (e.g. markers on the floor). Only the first few frames of new poses will be applied to the model. This will result in more stable tracking.
+     * @default false
+    */
+    imageDoesNotMove = false;
+    /**
+     * Enable to hide the tracked object when the image is not tracked anymore. When disabled the tracked object will stay at the position it was last tracked at.
+     * @default true
+     */
+    hideWhenTrackingIsLost = true;
+    /**
+     * Get the name of the image file from the url
+     */
+    getNameFromUrl() {
+        if (this.image) {
+            const parts = this.image.split("/");
+            return parts[parts.length - 1];
+        }
+        return null;
+    }
+}
+__decorate([
+    serializable(URL)
+], WebXRImageTrackingModel.prototype, "image", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "widthInMeters", void 0);
+__decorate([
+    serializable(AssetReference)
+], WebXRImageTrackingModel.prototype, "object", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "createObjectInstance", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "imageDoesNotMove", void 0);
+__decorate([
+    serializable()
+], WebXRImageTrackingModel.prototype, "hideWhenTrackingIsLost", void 0);
+class ImageTrackingExtension {
+    exporter;
+    component;
+    isImageTrackingExtension = true;
+    get extensionName() { return "image-tracking"; }
+    constructor(exporter, component) {
+        this.exporter = exporter;
+        this.component = component;
+        if (debug)
+            console.log(this);
+        this.exporter.anchoringType = "image";
+    }
+    // set during export
+    shouldExport = true;
+    filename = null;
+    imageModel = null;
+    onBeforeBuildDocument(_context) {
+        // check if this extension is the first image tracking extension in the list
+        // since iOS can only track one image at a time we only allow one image tracking extension to be active
+        // we have to determine this at the earlierst export callback
+        // all subsequent export callbacks should then check is shouldExport is set to true
+        // this should only be the case for exactly one extension
+        const index = this.exporter.extensions
+            .filter(e => {
+            const ext = e;
+            return ext.isImageTrackingExtension && ext.component.activeAndEnabled && ext.component.trackedImages?.length > 0;
+        })
+            .indexOf(this);
+        this.shouldExport = index === 0;
+        if (!this.shouldExport)
+            return;
+        // Warn if more than one tracked image is used for USDZ; that's not supported at the moment.
+        if (this.component.trackedImages?.length > 1) {
+            if (debug || isDevEnvironment()) {
+                showBalloonWarning("USDZ: Only one tracked image is supported.");
+                console.warn("USDZ: Only one tracked image is supported. Will choose the first one in the trackedImages list");
+            }
+        }
+    }
+    onAfterHierarchy(_context, writer) {
+        if (!this.shouldExport)
+            return;
+        const iOSVersion = DeviceUtilities.getiOSVersion();
+        const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18;
+        const workaroundForFB16119331 = majorVersion >= 18;
+        const multiplier = workaroundForFB16119331 ? 1 : 100;
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.imageModel.widthInMeters * multiplier).toFixed(8));
+        writer.closeBlock();
+    }
+    async onAfterSerialize(context) {
+        if (!this.shouldExport)
+            return;
+        const imageModel = this.imageModel;
+        const img = _imageElements.get(imageModel.image);
+        const canvas = await imageToCanvas(img);
+        const blob = await canvas.convertToBlob({ type: 'image/png' });
+        const arrayBuffer = await blob.arrayBuffer();
+        context.files['image_tracking/' + this.filename] = new Uint8Array(arrayBuffer);
+    }
+    onExportObject(object, model, _context) {
+        if (!this.shouldExport)
+            return;
+        const imageTracking = this.component;
+        if (!imageTracking || !imageTracking.trackedImages?.length || !imageTracking.activeAndEnabled)
+            return;
+        // we only care about the first image
+        // We can only apply this to the first tracked image, more are not supported by QuickLook.
+        const trackedImage = imageTracking.trackedImages[0];
+        if (trackedImage.object?.asset === object) {
+            this.imageModel = trackedImage;
+            this.filename = trackedImage.getNameFromUrl() || "marker.png";
+            const { scale, target } = this.exporter.getARScaleAndTarget();
+            // We have to reset the image tracking object's position and rotation, because QuickLook applies them.
+            // On Android WebXR they're replaced by the tracked data
+            let parent = object;
+            const relativeMatrix = new Matrix4();
+            if (object !== target) {
+                while (parent && parent.parent && parent.parent !== target) {
+                    parent = parent.parent;
+                    relativeMatrix.premultiply(parent.matrix);
+                }
+            }
+            const mat = relativeMatrix
+                .clone()
+                .invert();
+            // apply session root scale again after undoing the world transformation
+            model.setMatrix(mat.scale(new Vector3(scale, scale, scale)));
+            // Unfortunately looks like Apple's docs are incomplete:
+            // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims
+            // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects.
+            // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker".
+            // model.extraSchemas.push("Preliminary_AnchoringAPI");
+            // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => {
+            // writer.appendLine( `token preliminary:anchoring:type = "image"` );
+            // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
+            // });
+        }
+    }
+}
+/**
+ * Add this component to a object to enable image tracking in WebXR sessions.
+ *
+ * You need to add at least one {@link WebXRImageTrackingModel} to the `trackedImages` array to define which images to track and which objects to place on top of them.
+ *
+ * **NOTE:** For Android devices, image tracking currently requires the user to enable the `chrome://flags/#webxr-incubations` flag in Chrome.
+ *
+ * **NOTE:** For iOS only one image can be tracked at a time. If you have multiple images in the `trackedImages` array, only the first one will be tracked. You can use the {@link setPrimaryImage} method to change which image is tracked before entering the XR session.
+ *
+ * - Example: https://samples.needle.tools/image-tracking
+ *
+ * @category XR
+ * @group Components
+ */
+export class WebXRImageTracking extends Behaviour {
+    /**
+     * If you have multiple images in your application to track then this method is useful for iOS AR where only the first image can be tracked.
+     * Call this method will set the given image as the first one in the array so it will be used for tracking.
+     */
+    setPrimaryImage(image) {
+        const index = this.trackedImages.indexOf(image);
+        if (index >= 0) {
+            const current = this.trackedImages[0];
+            if (current !== image) {
+                this.trackedImages[0] = image;
+                this.trackedImages[index] = current;
+            }
+        }
+        else
+            console.warn(`[WebXRImageTracking] Can not set primary: image not found in 'trackedImages' array ${image.image}`);
+    }
+    /**
+     * Add an image to track. If the image is already in the trackedImages array it won't be added again.
+     * Note: that adding images at runtime *while* in AR is not supported.
+     *
+     * @example
+     * ```ts
+     * const imageTracking = GameObject.getComponent(myGameObject, WebXRImageTracking);
+     * const image = new WebXRImageTrackingModel({ url: "https://example.com/my-marker.png", widthInMeters: 0.2, object: myObject });
+     * imageTracking.addImage(image, true); // add and set as primary image
+     * ```
+     */
+    addImage(image, asPrimary = false) {
+        if (!this.trackedImages.includes(image)) {
+            this.trackedImages.push(image);
+            loadImage(image.image);
+        }
+        if (asPrimary)
+            this.setPrimaryImage(image);
+    }
+    /**
+     * List of images to track in the WebXR session. Use {@link WebXRImageTrackingModel} to define each image and the object to place on top of it.
+     *
+     * Use the `addImage()` and `setPrimaryImage()` methods to modify this array at runtime.
+     */
+    trackedImages = [];
+    /** Applies smoothing based on detected jitter to the tracked image. */
+    smooth = true;
+    trackedImageIndexMap = new Map();
+    /** @returns true if image tracking is supported on this device. This may return false at runtime if the user's browser did not enable webxr incubations */
+    get supported() { return this._supported; }
+    _supported = true;
+    /** @internal */
+    awake() {
+        if (debug)
+            console.log(this);
+        if (!this.trackedImages)
+            return;
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image) {
+                loadImage(trackedImage.image);
+            }
+        }
+    }
+    /** @internal */
+    onEnable() {
+        USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport);
+    }
+    /** @internal */
+    onDisable() {
+        USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport);
+    }
+    onBeforeUSDZExport = (args) => {
+        if (this.activeAndEnabled && this.trackedImages?.length) {
+            args.exporter.extensions.push(new ImageTrackingExtension(args.exporter, this));
+        }
+    };
+    /** @internal */
+    onBeforeXR(_mode, args) {
+        // console.log("onXRRequested", args, this.trackedImages)
+        if (this.trackedImages) {
+            args.optionalFeatures = args.optionalFeatures || [];
+            if (!args.optionalFeatures.includes("image-tracking"))
+                args.optionalFeatures.push("image-tracking");
+            args.trackedImages = [];
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
+                    const bitmap = _imageElements.get(trackedImage.image);
+                    if (bitmap) {
+                        this.trackedImageIndexMap.set(args.trackedImages.length, trackedImage);
+                        args.trackedImages.push({
+                            image: bitmap,
+                            widthInMeters: trackedImage.widthInMeters
+                        });
+                    }
+                }
+            }
+        }
+    }
+    /** @internal */
+    onEnterXR(_args) {
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    // capture the initial state of tracked images in the scene to restore them when the session ends
+                    const obj = trackedImage.object.asset;
+                    if (!obj.userData)
+                        obj.userData = {};
+                    const state = {
+                        visible: obj.visible,
+                        parent: obj.parent,
+                        matrix: obj.matrix.clone()
+                    };
+                    obj.userData["image-tracking"] = state;
+                }
+            }
+        }
+        // clear out all frame counters for tracking
+        for (const trackedData of this.imageToObjectMap.values()) {
+            trackedData.frames = 0;
+        }
+    }
+    ;
+    /** @internal */
+    onLeaveXR(_args) {
+        if (!this.supported && DeviceUtilities.isAndroidDevice()) {
+            showBalloonWarning(this.webXRIncubationsWarning);
+        }
+        if (this.trackedImages) {
+            for (const trackedImage of this.trackedImages) {
+                if (trackedImage.object?.asset) {
+                    const obj = trackedImage.object.asset;
+                    if (obj.userData) {
+                        // restore the initial state of tracked images in the scene
+                        const state = obj.userData["image-tracking"];
+                        if (state) {
+                            obj.visible = state.visible;
+                            state.parent?.add(obj);
+                            obj.matrix.copy(state.matrix);
+                            obj.matrix.decompose(obj.position, obj.quaternion, obj.scale);
+                        }
+                        delete obj.userData["image-tracking"];
+                    }
+                }
+            }
+        }
+    }
+    imageToObjectMap = new Map();
+    currentImages = [];
+    webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
+    /** @internal */
+    onUpdateXR(args) {
+        this.currentImages.length = 0;
+        const frame = args.xr.frame;
+        if (!frame)
+            return;
+        if (!("getImageTrackingResults" in frame)) {
+            if (!this["didPrintWarning"]) {
+                this["didPrintWarning"] = true;
+                console.log(this.webXRIncubationsWarning);
+            }
+            this._supported = false;
+            showBalloonWarning(this.webXRIncubationsWarning);
+            return;
+        }
+        // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch
+        // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled
+        else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) {
+            // Image tracking is not enabled for this session
+            return;
+        }
+        else if (frame.session && typeof frame.getImageTrackingResults === "function") {
+            const results = frame.getImageTrackingResults();
+            if (results.length > 0) {
+                const space = this.context.renderer.xr.getReferenceSpace();
+                if (space) {
+                    for (const result of results) {
+                        const state = result.trackingState;
+                        const imageIndex = result.index;
+                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
+                        if (trackedImage) {
+                            const pose = frame.getPose(result.imageSpace, space);
+                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
+                            this.currentImages.push(imageData);
+                        }
+                        else {
+                            if (debug) {
+                                console.warn("No tracked image for index", imageIndex);
+                            }
+                        }
+                    }
+                    if (this.currentImages.length > 0) {
+                        try {
+                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: this.currentImages }));
+                            this.onImageTrackingUpdate(this.currentImages);
+                        }
+                        catch (e) {
+                            console.error(e);
+                        }
+                    }
+                }
+            }
+        }
+        // disable any objects that are no longer tracked
+        /** time in millis */
+        const hysteresis = 1000;
+        for (const [key, value] of this.imageToObjectMap) {
+            if (!value.object || !key)
+                continue;
+            // If the user disallowed hiding the object when tracking is lost, skip this
+            if (key.hideWhenTrackingIsLost === false)
+                continue;
+            let found = false;
+            for (const trackedImage of this.currentImages) {
+                if (trackedImage.model === key) {
+                    // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device)
+                    const timeSinceLastTracking = Date.now() - value.lastTrackingTime;
+                    if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            if (!found) {
+                GameObject.setActive(value.object, false);
+            }
+        }
+    }
+    onImageTrackingUpdate = (images) => {
+        const xr = NeedleXRSession.active;
+        if (!xr)
+            return;
+        for (const image of images) {
+            const model = image.model;
+            const isTracked = image.state === "tracked";
+            // don't do anything if we don't have an object to track - can be handled externally through events
+            if (!model.object)
+                continue;
+            let trackedData = this.imageToObjectMap.get(model);
+            if (trackedData === undefined) {
+                trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() };
+                this.imageToObjectMap.set(model, trackedData);
+                model.object.loadAssetAsync().then((asset) => {
+                    if (model.createObjectInstance && asset) {
+                        asset = GameObject.instantiate(asset);
+                    }
+                    if (asset) {
+                        trackedData.object = asset;
+                        // workaround for instancing currently not properly updating 
+                        // instanced objects become visible when the image is recognized for the second time
+                        // we need to look into this further https://linear.app/needle/issue/NE-3936
+                        for (const rend of asset.getComponentsInChildren(Renderer)) {
+                            rend.setInstancingEnabled(false);
+                        }
+                        // make sure to parent to the WebXR.rig
+                        if (xr.rig) {
+                            xr.rig.gameObject.add(asset);
+                            image.applyToObject(asset);
+                            if (!asset.activeSelf)
+                                GameObject.setActive(asset, true);
+                            // InstancingUtil.markDirty(asset);
+                        }
+                        else {
+                            console.warn("XRImageTracking: missing XRRig");
+                        }
+                    }
+                });
+            }
+            else {
+                trackedData.frames++;
+                if (isTracked)
+                    trackedData.lastTrackingTime = Date.now();
+                // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data
+                // to improve the tracking quality a bit.
+                if (model.imageDoesNotMove && trackedData.frames > 10)
+                    continue;
+                if (!trackedData.object)
+                    continue;
+                if (xr.rig) {
+                    xr.rig.gameObject.add(trackedData.object);
+                    image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined);
+                    if (!trackedData.object.activeSelf) {
+                        GameObject.setActive(trackedData.object, true);
+                    }
+                    // InstancingUtil.markDirty(trackedData.object);
+                }
+            }
+        }
+    };
+}
+__decorate([
+    serializable(WebXRImageTrackingModel)
+], WebXRImageTracking.prototype, "trackedImages", void 0);
+__decorate([
+    serializable()
+], WebXRImageTracking.prototype, "smooth", void 0);
+const _imageElements = new Map();
+const _imageLoadingPromises = new Map();
+async function loadImage(url) {
+    if (_imageElements.has(url)) {
+        if (_imageLoadingPromises.has(url))
+            return _imageLoadingPromises.get(url);
+        return Promise.resolve(true);
+    }
+    const promise = new Promise(res => {
+        _imageElements.set(url, null);
+        const imageElement = document.createElement("img");
+        imageElement.src = url;
+        imageElement.addEventListener("load", async () => {
+            const img = await createImageBitmap(imageElement);
+            _imageElements.set(url, img);
+            res(true);
+        });
+    });
+    _imageLoadingPromises.set(url, promise);
+    promise.finally(() => {
+        _imageLoadingPromises.delete(url);
+    });
+    return promise;
+}
 //# sourceMappingURL=WebXRImageTracking.js.map