@needle-tools/engine 3.3.0-alpha → 3.5.0-alpha

package/src/engine-components/webxr/WebXRImageTracking.ts

@@ -1,9 +1,13 @@
-import { WebXR } from "./WebXR";
+import { WebXR, WebXREvent } from "./WebXR";
 import { serializable } from "../../engine/engine_serialization";
 import { Behaviour, GameObject } from "../Component";
 import { Object3D, Quaternion, Vector3 } from "three";
 import { CircularBuffer, getParam } from "../../engine/engine_utils";
 import { AssetReference } from "../../engine/engine_addressables";
+import { showBalloonWarning } from "../../engine/debug";
+
+import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter";
+import { USDZExporterContext, USDWriter, imageToCanvas } from "../../engine-components/export/usdz/ThreeUSDZExporter";
 
 // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
 
@@ -18,16 +22,6 @@ export class WebXRTrackedImage {
     readonly measuredSize: number;
     readonly state: "tracked" | "emulated";
 
-    // private _matrix: Matrix4 | null = null;
-    // private get matrix(): Matrix4 {
-    //     if (!this._matrix) {
-    //         // this._matrix = WebXRTrackedImage._matrixBuffer.get();
-    //         // const matrix = this._pose.transform.matrix;
-    //         // this._matrix.fromArray(matrix);
-    //     }
-    //     return this._matrix!;
-    // }
-
     /** Copy the image position to a vector */
     getPosition(vec: Vector3) {
         this.ensureTransformData();
@@ -42,13 +36,23 @@ export class WebXRTrackedImage {
         return quat;
     }
 
-    applyToObject(object: Object3D) {
+    private static y180 = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);
+    applyToObject(object: Object3D, t01: number | undefined = undefined) {
         this.ensureTransformData();
-        object.position.copy(this._position);
-        object.quaternion.copy(this._rotation);
+        // check if position/_position or rotation/_rotation changed more than just a little bit
+        const haveChanged = object.position.distanceToSquared(this._position) > 0.05 || object.quaternion.angleTo(this._rotation) > 0.05;
+        if (t01 === undefined || t01 >= 1 || haveChanged) {
+            object.position.copy(this._position);
+            object.quaternion.copy(this._rotation);
+        }
+        else {
+            t01 = Math.max(0, Math.min(1, t01));
+            object.position.lerp(this._position, t01);
+            object.quaternion.slerp(this._rotation, t01);
+        }
+        object.quaternion.multiply(WebXRTrackedImage.y180);
     }
 
-    // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
     private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
     private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
     private _position!: Vector3;
@@ -58,8 +62,14 @@ export class WebXRTrackedImage {
             this._position = WebXRTrackedImage._positionBuffer.get();
             this._rotation = WebXRTrackedImage._rotationBuffer.get();
             const t = this._pose.transform;
-            this._position.set(-t.position.x, t.position.y, -t.position.z);
-            this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+
+            // when parented to the world, we need to flip data here
+            //this._position.set(-t.position.x, t.position.y, -t.position.z);
+            // this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+
+            // for some reason when parented to the XRRig, we need the original data
+            this._position.set(t.position.x, t.position.y, t.position.z);
+            this._rotation.set(t.orientation.x, t.orientation.y, t.orientation.z, t.orientation.w);
         }
     }
 
@@ -99,6 +109,32 @@ export class WebXRImageTrackingModel {
     imageDoesNotMove: boolean = false;
 }
 
+class ImageTrackingExtension {
+
+    get extensionName() { return "image-tracking"; }
+
+    private filename: string;
+    private widthInMeters: number;
+    private imageData: Uint8Array;
+
+    constructor(filename: string, imageData: Uint8Array, widthInMeters: number) {
+        this.filename = filename;
+        this.imageData = imageData;
+        this.widthInMeters = widthInMeters;
+    }
+
+    onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`);
+        writer.appendLine(`uniform asset image = @tracker/` + this.filename + `@`);
+        writer.appendLine(`uniform double physicalWidth = ` + (this.widthInMeters * 100).toFixed(8));
+        writer.closeBlock();
+    }
+
+    onAfterSerialize(context: USDZExporterContext) {
+        context.files['tracker/' + this.filename] = this.imageData;
+    }
+}
+
 export class WebXRImageTracking extends Behaviour {
 
     @serializable(WebXRImageTrackingModel)
@@ -121,22 +157,43 @@ export class WebXRImageTracking extends Behaviour {
                     imageElement.addEventListener("load", async () => {
                         const img = await createImageBitmap(imageElement);
                         WebXRImageTracking._imageElements.set(url, img);
+
+                        // read back Uint8Array to use in USDZ - 
+                        // TODO better would be to do that once we actually need it
+                        const canvas = await imageToCanvas( img );
+                        if (canvas) {
+                            const blob = await new Promise( resolve => canvas.toBlob( resolve, 'image/png', 1 ) ) as any;
+                            const arrayBuffer = await blob.arrayBuffer();
+                            
+                            const exporter = GameObject.findObjectOfType(USDZExporter);
+                            if (exporter) {
+                                exporter.extensions.push(
+                                    new ImageTrackingExtension("marker.png", new Uint8Array(arrayBuffer), this.trackedImages[0].widthInMeters) 
+                                );                           
+                                exporter.anchoringType = "image";
+                            }
+                        } 
                     });
                 }
             }
         }
     }
 
+    private xr: WebXR | null = null;
+
     onEnable(): void {
-        WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
-        WebXR.addEventListener("xrStarted", this.onXRStarted);
-        this.addEventListener("image-tracking", this.onImageTrackingUpdate)
+        this.xr = GameObject.findObjectOfType(WebXR);
+        WebXR.addEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.addEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.addEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.addEventListener("image-tracking", this.onImageTrackingUpdate);
     }
 
     onDisable(): void {
-        WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
-        WebXR.removeEventListener("xrStarted", this.onXRStarted);
-        this.removeEventListener("image-tracking", this.onImageTrackingUpdate)
+        WebXR.removeEventListener(WebXREvent.ModifyAROptions, this.onModifyAROptions);
+        WebXR.removeEventListener(WebXREvent.XRStarted, this.onXRStarted);
+        WebXR.removeEventListener(WebXREvent.XRUpdate, this.onXRUpdate);
+        this.removeEventListener("image-tracking", this.onImageTrackingUpdate);
     }
 
     private onModifyAROptions = (event: any) => {
@@ -197,8 +254,12 @@ export class WebXRImageTracking extends Behaviour {
 
                     if (asset) {
                         trackedData!.object = asset;
-                        if (asset !== this.gameObject)
-                            this.gameObject.add(asset);
+                        
+                        // make sure to parent to the WebXR.rig
+                        if (this.xr) {
+                            this.xr.Rig.add(asset);
+                        }
+
                         image.applyToObject(asset);
                         if (!asset.activeSelf)
                             GameObject.setActive(asset, true);
@@ -215,6 +276,10 @@ export class WebXRImageTracking extends Behaviour {
                 
                 if (!trackedData.object) continue;
                 
+                if (this.xr) {
+                    this.xr.Rig.add(trackedData.object);
+                }
+
                 image.applyToObject(trackedData.object);
                 if (!trackedData.object.activeSelf)
                     GameObject.setActive(trackedData.object, true);
@@ -229,9 +294,17 @@ export class WebXRImageTracking extends Behaviour {
         }
     };
 
-    onBeforeRender(frame: XRFrame | null): void {
+    private onXRUpdate = (evt): void => {
+        const frame = evt.frame;
+        if (!frame) return;
+
+        if (frame.session && !("getImageTrackingResults" in frame)) {
+            showBalloonWarning("Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a href=\"chrome://flags/#webxr-incubations\">chrome://flags/#webxr-incubations</a> flag.");
+            return;
+        }
+
         //@ts-ignore
-        if (frame?.session && typeof frame.getImageTrackingResults === "function") {
+        if (frame.session && typeof frame.getImageTrackingResults === "function") {
             //@ts-ignore
             const results = frame.getImageTrackingResults();
             if (results.length) {

package/lib/engine-components/export/usdz/types.d.ts

@@ -1,34 +0,0 @@
-import { Object3D, Matrix4, Material, BufferGeometry } from "three";
-/** implementation is in three */
-export declare class USDZDocument {
-    name: string;
-    get isDocumentRoot(): boolean;
-    add(obj: USDZObject): any;
-    remove(obj: USDZObject): any;
-    traverse(callback: (obj: USDZObject) => void): any;
-    findById(uuid: string): USDZObject | undefined;
-    get isDynamic(): boolean;
-}
-/** implementation is in three */
-export declare class USDZObject {
-    static createEmptyParent(parent: USDZObject): any;
-    uuid: string;
-    name: string;
-    matrix: Matrix4;
-    material: Material;
-    geometry: BufferGeometry;
-    parent: USDZObject | USDZDocument | null;
-    children: USDZObject[];
-    _eventListeners: {
-        [event: string]: Function[];
-    };
-    isDynamic: boolean;
-    is(obj: Object3D): boolean;
-    isEmpty(): boolean;
-    clone(): any;
-    getPath(): any;
-    add(child: USDZObject): any;
-    remove(child: USDZObject): any;
-    addEventListener(evt: string, listener: Function): any;
-    removeEventListener(evt: string, listener: Function): any;
-}

package/lib/engine-components/export/usdz/types.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=types.js.map

package/lib/engine-components/export/usdz/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/engine-components/export/usdz/types.ts"],"names":[],"mappings":""}

package/src/engine-components/export/usdz/types.ts

@@ -1,39 +0,0 @@
-import { Object3D, Matrix4, Material, BufferGeometry } from "three";
-
-// keep in sync with USDZExporter.js
-
-/** implementation is in three */
-export declare class USDZDocument {
-    name: string;
-    get isDocumentRoot(): boolean;
-    add(obj: USDZObject);
-    remove(obj: USDZObject);
-    traverse(callback: (obj: USDZObject) => void);
-    findById(uuid: string): USDZObject | undefined;
-    get isDynamic(): boolean;
-}
-
-
-
-/** implementation is in three */
-export declare class USDZObject {
-    static createEmptyParent(parent: USDZObject);
-    uuid: string;
-    name: string;
-    matrix: Matrix4;
-    material: Material;
-    geometry: BufferGeometry;
-    parent: USDZObject | USDZDocument | null;
-    children: USDZObject[];
-    _eventListeners: { [event: string]: Function[] };
-    isDynamic: boolean;
-
-    is(obj: Object3D): boolean;
-    isEmpty(): boolean;
-    clone();
-    getPath();
-    add(child: USDZObject);
-    remove(child: USDZObject);
-    addEventListener(evt: string, listener: Function);
-    removeEventListener(evt: string, listener: Function);
-}