@needle-tools/engine 2.51.0-pre → 2.53.0-pre

package/src/engine-components/SpriteRenderer.ts

@@ -1,9 +1,48 @@
 import { Behaviour } from "./Component";
 import * as THREE from "three";
-import { serializable } from "../engine/engine_serialization_decorator";
-import { Color, Material, Texture } from "three";
+import { serializable, serializeable } from "../engine/engine_serialization_decorator";
+import { Color, Material, NearestFilter, Texture, Vector2, TextureFilter } from "three";
 import { RGBAColor } from "./js-extensions/RGBAColor";
+import { getParam } from "../engine/engine_utils";
 
+const debug = getParam("debugspriterenderer")
+
+class SpriteUtils {
+
+    static cache: { [key: string]: THREE.BufferGeometry } = {};
+
+    static getOrCreateGeometry(sprite: Sprite): THREE.BufferGeometry {
+        if (sprite._geometry) return sprite._geometry;
+        if (sprite.guid) {
+            if (SpriteUtils.cache[sprite.guid]) {
+                if (debug) console.log("Take cached geometry for sprite", sprite.guid);
+                return SpriteUtils.cache[sprite.guid];
+            }
+        }
+        const geo = new THREE.BufferGeometry();
+        sprite._geometry = geo;
+        const vertices = new Float32Array(sprite.triangles.length * 3);
+        const uvs = new Float32Array(sprite.triangles.length * 2);
+        for (let i = 0; i < sprite.triangles.length; i += 1) {
+            const index = sprite.triangles[i];
+
+            vertices[i * 3] = -sprite.vertices[index].x;
+            vertices[i * 3 + 1] = sprite.vertices[index].y;
+
+            vertices[i * 3 + 2] = 0;
+            const uv = sprite.uv[index];
+            uvs[i * 2] = uv.x;
+            uvs[i * 2 + 1] = 1 - uv.y;
+        }
+        geo.setAttribute("position", new THREE.BufferAttribute(vertices, 3));
+        geo.setAttribute("uv", new THREE.BufferAttribute(uvs, 2));
+        if (sprite.guid)
+            this.cache[sprite.guid] = geo;
+        if (debug)
+            console.log("Built sprite geometry", sprite, geo);
+        return geo;
+    }
+}
 
 /// <summary>
 ///   <para>SpriteRenderer draw mode.</para>
@@ -30,42 +69,59 @@ class Vec2 {
 
 export class Sprite {
 
+    @serializable()
+    guid?: string;
     @serializable(Texture)
     texture?: THREE.Texture;
-
+    @serializeable()
     triangles!: Array<number>;
+    @serializeable()
     uv!: Array<Vec2>;
+    @serializeable()
     vertices!: Array<Vec2>;
 
     _geometry?: THREE.BufferGeometry;
 }
 
-class SpriteUtils {
 
-    static getOrCreateGeometry(sprite: Sprite): THREE.BufferGeometry {
-        if (sprite._geometry) return sprite._geometry;
-        const geo = new THREE.BufferGeometry();
-        sprite._geometry = geo;
-        const vertices = new Float32Array(sprite.triangles.length * 3);
-        const uvs = new Float32Array(sprite.triangles.length * 2);
-        for (let i = 0; i < sprite.triangles.length; i += 1) {
-            const index = sprite.triangles[i];
+class Slice {
+    @serializable()
+    name!: string;
+    @serializable(Vector2)
+    offset!: Vector2;
+    @serializable(Vector2)
+    size!: Vector2;
+}
 
-            vertices[i * 3] = -sprite.vertices[index].x;
-            vertices[i * 3 + 1] = sprite.vertices[index].y;
+const $spriteTexOwner = Symbol("spriteOwner");
 
-            vertices[i * 3 + 2] = 0;
-            const uv = sprite.uv[index];
-            uvs[i * 2] = uv.x;
-            uvs[i * 2 + 1] = 1 - uv.y;
-        }
-        // console.log(vertices);
-        geo.setAttribute("position", new THREE.BufferAttribute(vertices, 3));
-        geo.setAttribute("uv", new THREE.BufferAttribute(uvs, 2));
-        return geo;
+export class SpriteSheet {
+
+    @serializable(Sprite)
+    sprite?: Sprite;
+    @serializable()
+    index: number = 0;
+    @serializable(Slice)
+    slices!: Slice[];
+
+    update() {
+        const index = this.index;
+        if (index < 0 || index >= this.slices.length)
+            return;
+        const slice = this.slices[index];
+        let tex = this.sprite?.texture;
+        if (!tex) return;
+        tex.encoding = THREE.sRGBEncoding;
+        tex.offset.set(slice.offset.x, slice.offset.y);
+        // aniso > 1 makes the texture blurry
+        if (tex.minFilter == NearestFilter && tex.magFilter == NearestFilter)
+            tex.anisotropy = 1;
+        // tex.repeat.set(slice.size.x, -slice.size.y);
+        tex.needsUpdate = true;
     }
 }
 
+
 export class SpriteRenderer extends Behaviour {
 
     @serializable()
@@ -79,37 +135,69 @@ export class SpriteRenderer extends Behaviour {
     @serializable(Material)
     sharedMaterial?: THREE.Material;
 
-    get sprite(): Sprite | undefined {
-        return this._sprite;
+    @serializable(SpriteSheet)
+    get sprite(): SpriteSheet | undefined {
+        return this._spriteSheet;
     }
-    set sprite(value: Sprite | undefined) {
-        if(value === this._sprite) return;
-        this._sprite = value;
-        this.updateSprite();
+    set sprite(value: SpriteSheet | undefined | number) {
+        if (value === this._spriteSheet) return;
+        if (typeof value === "number") {
+            const index = Math.floor(value);;
+            if (index === value)
+                this.spriteIndex = index;
+            return;
+        }
+        else {
+            this._spriteSheet = value;
+            this.updateSprite();
+        }
     }
 
-    private _sprite?: Sprite;
+    set spriteIndex(value: number) {
+        if (!this._spriteSheet) return;
+        this._spriteSheet.index = value;
+        this._spriteSheet.update();
+    }
+    get spriteIndex(): number {
+        return this._spriteSheet?.index ?? 0;
+    }
+    get spriteFrames(): number {
+        return this._spriteSheet?.slices.length ?? 0;
+    }
+
+    private _spriteSheet?: SpriteSheet;
     private _currentSprite?: THREE.Mesh;
 
     awake(): void {
         this._currentSprite = undefined;
+        if(debug) {
+            console.log("Awake", this.name, this, this.sprite?.sprite?.texture);
+            if(this.sprite?.sprite?.texture)
+                console.log(this.sprite.sprite.texture.minFilter.toString(), this.sprite.sprite.texture.magFilter.toString());
+        }
     }
 
     start() {
-        if (this.drawMode === SpriteDrawMode.Tiled) {
-            console.warn("Tiled draw mode is not supported yet", this);
-        }
         if (!this._currentSprite)
             this.updateSprite();
         else if (this.gameObject)
             this.gameObject.add(this._currentSprite);
     }
 
+    // frame : number = 0;
+    // update(){
+    //     // const frameRate = 12;
+    //     // this.frame += frameRate * this.context.time.deltaTime;
+    //     // if(this.frame >= this.spriteFrames)
+    //     //     this.frame = 0;
+    //     // this.spriteIndex = Math.floor(this.frame);
+    //     // console.log(this.spriteIndex);
+    // }
+
     private updateSprite() {
         if (!this.__didAwake) return;
-        if (!this.sprite) return;
-        if (!this.sharedMaterial) return;
-
+        if (!this.sprite?.sprite) return;
+        const sprite = this.sprite.sprite;
         if (!this._currentSprite) {
             const mat = new THREE.MeshBasicMaterial({ color: 0xffffff, side: THREE.DoubleSide });
             if (!mat) return;
@@ -120,17 +208,23 @@ export class SpriteRenderer extends Behaviour {
             }
             mat.alphaTest = 0.5;
 
-            if (this.sprite.texture && !mat.wireframe) {
-                const t = this.sprite.texture;
-                // t.wrapS = THREE.RepeatWrapping;
-                mat["map"] = t;
+            if (sprite.texture && !mat.wireframe) {
+                let tex = sprite.texture;
+                // the sprite renderer modifies the textue offset 
+                // so we need to clone the texture 
+                // if the same texture is used multiple times
+                if (tex[$spriteTexOwner] !== undefined && tex[$spriteTexOwner] !== this && this.spriteFrames > 1) {
+                    tex = sprite!.texture = tex.clone();
+                }
+                tex[$spriteTexOwner] = this;
+                mat["map"] = tex;
             }
-
-            this._currentSprite = new THREE.Mesh(SpriteUtils.getOrCreateGeometry(this.sprite), mat);
+            this.sharedMaterial = mat;
+            this._currentSprite = new THREE.Mesh(SpriteUtils.getOrCreateGeometry(sprite), mat);
         }
         else {
-            this._currentSprite.geometry = SpriteUtils.getOrCreateGeometry(this.sprite);
-            this._currentSprite.material["map"] = this.sprite.texture;
+            this._currentSprite.geometry = SpriteUtils.getOrCreateGeometry(sprite);
+            this._currentSprite.material["map"] = sprite.texture;
         }
 
         if (this._currentSprite.parent !== this.gameObject) {
@@ -139,5 +233,7 @@ export class SpriteRenderer extends Behaviour {
             if (this.gameObject)
                 this.gameObject.add(this._currentSprite);
         }
+
+        this._spriteSheet?.update();
     }
 }

package/src/engine-components/Voip.ts

@@ -307,6 +307,11 @@ export class Voip extends Behaviour {
             return;
         }
 
+        if (utils.isiOS() && utils.isSafari()) {
+            console.log("VOIP is currently not supported on Safari iOS")
+            return;
+        }
+
         this.peer = new Peer();
         navigator["getUserMedia"] = (navigator["getUserMedia"] || navigator["webkitGetUserMedia"] || navigator["mozGetUserMedia"] || navigator["msGetUserMedia"]);
 
@@ -360,10 +365,15 @@ export class Voip extends Behaviour {
     onDisable(): void {
         console.log("TODO: close all");
         for (const key in this.currentIncomingCalls) {
-            const call = this.currentIncomingCalls[key];
-            call?.close();
-            const con = this.connections[key];
-            con?.close();
+            try {
+                const call = this.currentIncomingCalls[key];
+                call?.close();
+                const con = this.connections[key];
+                con?.close();
+            }
+            catch (err) {
+                console.error(err);
+            }
         }
     }
 

package/src/engine-components/WebXRRig.ts

@@ -1,9 +1,22 @@
-import { Behaviour } from "./Component";
+import { Object3D } from "three";
+import { IGameObject } from "../engine/engine_types";
+import { getParam } from "../engine/engine_utils";
+import { Behaviour, GameObject } from "./Component";
+import { BoxGizmo } from "./Gizmos";
 
+const debug = getParam("debugrig");
 
 export class XRRig extends Behaviour {
     awake(): void {
         // const helper = new AxesHelper(.1);
         // this.gameObject.add(helper);
+        if (debug) {
+            const gizmoObj = new Object3D() as IGameObject;
+            gizmoObj.position.y += .5;
+            this.gameObject.add(gizmoObj);
+            const gizmo = gizmoObj.addNewComponent(BoxGizmo);
+            if (gizmo)
+                gizmo.isGizmo = false;
+        }
     }
 }

package/src/engine-components/codegen/components.ts

@@ -85,6 +85,7 @@ export { SpatialTriggerReceiver } from "../SpatialTrigger";
 export { SpatialTrigger } from "../SpatialTrigger";
 export { SpectatorCamera } from "../SpectatorCamera";
 export { Sprite } from "../SpriteRenderer";
+export { SpriteSheet } from "../SpriteRenderer";
 export { SpriteRenderer } from "../SpriteRenderer";
 export { SyncedCamera } from "../SyncedCamera";
 export { SyncedRoom } from "../SyncedRoom";
@@ -122,8 +123,6 @@ export { Avatar_Brain_LookAt } from "../avatar/Avatar_Brain_LookAt";
 export { Avatar_MouthShapes } from "../avatar/Avatar_MouthShapes";
 export { Avatar_MustacheShake } from "../avatar/Avatar_MustacheShake";
 export { LogStats } from "../debug/LogStats";
-export { GltfExportBox } from "../export/GltfExport";
-export { GltfExport } from "../export/GltfExport";
 export { RGBAColor } from "../js-extensions/RGBAColor";
 export { PlayableDirector } from "../timeline/PlayableDirector";
 export { SignalAsset } from "../timeline/SignalAsset";
@@ -159,3 +158,9 @@ export { Rect } from "../ui/RectTransform";
 export { RectTransform } from "../ui/RectTransform";
 export { SpatialHtml } from "../ui/SpatialHtml";
 export { Text } from "../ui/Text";
+export { GltfExportBox } from "../export/gltf/GltfExport";
+export { GltfExport } from "../export/gltf/GltfExport";
+export { USDZExporter } from "../export/usdz/USDZExporter";
+export { RegisteredAnimationInfo } from "../export/usdz/extensions/Animation";
+export { TransformData } from "../export/usdz/extensions/Animation";
+export { AnimationExtension } from "../export/usdz/extensions/Animation";

package/src/engine-components/export/{GltfExport.ts → gltf/GltfExport.ts}

@@ -1,12 +1,12 @@
-import { Behaviour, GameObject } from "../Component";
+import { Behaviour, GameObject } from "../../Component";
 import { GLTFExporter } from 'three/examples/jsm/exporters/GLTFExporter.js';
-import GLTFMeshGPUInstancingExtension from '../../include/three/EXT_mesh_gpu_instancing_exporter.js';
-import { Renderer } from "../Renderer";
+import GLTFMeshGPUInstancingExtension from '../../../include/three/EXT_mesh_gpu_instancing_exporter.js';
+import { Renderer } from "../../Renderer";
 import { Object3D, Vector3 } from "three";
-import { SerializationContext } from "../../engine/engine_serialization_core";
-import { NEEDLE_components } from "../../engine/extensions/NEEDLE_components";
-import { getWorldPosition } from "../../engine/engine_three_utils";
-import { BoxHelperComponent } from "../BoxHelperComponent";
+import { SerializationContext } from "../../../engine/engine_serialization_core";
+import { NEEDLE_components } from "../../../engine/extensions/NEEDLE_components";
+import { getWorldPosition } from "../../../engine/engine_three_utils";
+import { BoxHelperComponent } from "../../BoxHelperComponent";
 import { AnimationClip } from "three";
 
 

package/src/engine-components/export/usdz/Extension.ts

@@ -0,0 +1,12 @@
+import { USDZObject } from "./types";
+
+
+export interface IUSDZExporterExtension {
+
+    get extensionName(): string;
+    onBeforeBuildDocument?(context);
+    onAfterBuildDocument?(context);
+    onExportObject?(object, model : USDZObject, context);
+    onAfterSerialize?(context);
+    onAfterHierarchy?(context);
+}

package/src/engine-components/export/usdz/USDZExporter.ts

@@ -0,0 +1,216 @@
+import { delay, getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils";
+import { Object3D, Color } from "three";
+import * as THREE from "three";
+import { USDZExporter as ThreeUSDZExporter } from "three/examples/jsm/exporters/USDZExporter";
+import { AnimationExtension } from "./extensions/Animation"
+import { ensureQuicklookLinkIsCreated } from "./utils/quicklook";
+import { getFormattedDate } from "./utils/timeutils";
+import { registerAnimatorsImplictly } from "./utils/animationutils";
+import { IUSDZExporterExtension } from "./Extension";
+import { Behaviour, GameObject } from "../../Component";
+import { WebXR } from "../../WebXR"
+import { serializable } from "../../../engine/engine_serialization";
+import { showBalloonWarning } from "../../../engine/debug/debug";
+import { Context } from "../../../engine/engine_setup";
+
+const debug = getParam("debugusdz");
+
+export type QuickLookOverlay = {
+    callToAction?: string;
+    checkoutTitle?: string;
+    checkoutSubtitle?: string;
+}
+
+export class USDZExporter extends Behaviour {
+
+    @serializable(Object3D)
+    objectToExport?: THREE.Object3D;
+
+    @serializable()
+    autoExportAnimations: boolean = false;
+
+    extensions: IUSDZExporterExtension[] = [];
+
+    private link!: HTMLAnchorElement;
+    private webxr?: WebXR;
+
+
+    start() {
+        if (debug) {
+            window.addEventListener("keydown", (evt) => {
+                switch (evt.key) {
+                    case "t":
+                        this.exportAsync();
+                        break;
+                }
+            });
+            if (isMobileDevice()) {
+                setTimeout(() => {
+                    this.exportAsync();
+                }, 2000)
+            }
+        }
+        document.getElementById("open-in-ar")?.addEventListener("click", (evt) => {
+            evt.preventDefault();
+            this.exportAsync();
+        });
+
+        if (!this.objectToExport) this.objectToExport = this.gameObject;
+    }
+
+
+
+    onEnable() {
+        const ios = isiOS()
+        const safari = isSafari();
+        if (debug || (ios && safari)) {
+            this.createQuicklookButton();
+            this.lastCallback = this.quicklookCallback.bind(this);
+            this.link = ensureQuicklookLinkIsCreated(this.context);
+            this.link.addEventListener('message', this.lastCallback);
+        }
+    }
+
+    onDisable() {
+        this.link?.removeEventListener('message', this.lastCallback);
+    }
+
+    async exportAsync() {
+        if (!this.objectToExport) return;
+
+        const exporter = new ThreeUSDZExporter();
+        const extensions: any = [...this.extensions]
+
+        // collect animators and their clips
+        const animExt = new AnimationExtension();
+        extensions.push(animExt);
+
+        if (this.autoExportAnimations)
+            registerAnimatorsImplictly(this.objectToExport, animExt);
+
+        const eventArgs = { self: this, exporter: exporter, extensions: extensions };
+        this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
+
+        let name = "needle";
+        if (debug) name += "-" + getFormattedDate();
+
+        //@ts-ignore
+        exporter.debug = debug;
+        //@ts-ignore
+        const arraybuffer = await exporter.parse(this.objectToExport, extensions);
+        const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
+
+        // second file: USDA (without assets)
+        //@ts-ignore
+        // const usda = exporter.lastUsda;
+        // const blob2 = new Blob([usda], { type: 'text/plain' });
+        // this.link.download = name + ".usda";
+        // this.link.href = URL.createObjectURL(blob2);
+        // this.link.click();
+
+        // see https://developer.apple.com/documentation/arkit/adding_an_apple_pay_button_or_a_custom_action_in_ar_quick_look
+        const overlay = this.buildQuicklookOverlay();
+        const callToAction = overlay.callToAction ? encodeURIComponent(overlay.callToAction) : "";
+        const checkoutTitle = overlay.checkoutTitle ? encodeURIComponent(overlay.checkoutTitle) : "";
+        const checkoutSubtitle = overlay.checkoutSubtitle ? encodeURIComponent(overlay.checkoutSubtitle) : "";
+        this.link.href = URL.createObjectURL(blob) + `#callToAction=${callToAction}&checkoutTitle=${checkoutTitle}&checkoutSubtitle=${checkoutSubtitle}&`;
+
+
+        if (!this.lastCallback) {
+            this.lastCallback = this.quicklookCallback.bind(this);
+            this.link.addEventListener('message', this.lastCallback);
+        }
+
+        // open quicklook
+        this.link.download = name + ".usdz";
+        this.link.click();
+
+        // TODO detect QuickLook availability:
+        // https://webkit.org/blog/8421/viewing-augmented-reality-assets-in-safari-for-ios/#:~:text=inside%20the%20anchor.-,Feature%20Detection,-To%20detect%20support
+    }
+
+    private lastCallback?: any;
+
+    private quicklookCallback(event) {
+        if ((event as any)?.data == '_apple_ar_quicklook_button_tapped') {
+            if (debug) showBalloonWarning("Quicklook closed via call to action button");
+            this.dispatchEvent(new CustomEvent("quicklook-button-tapped", { detail: this }));
+        }
+    }
+
+    private buildQuicklookOverlay(): QuickLookOverlay {
+        const obj: QuickLookOverlay = {};
+        obj.callToAction = "Close";
+        obj.checkoutTitle = "🌵 Made with Needle";
+        obj.checkoutSubtitle = "_";
+        // Use the quicklook-overlay event to customize the overlay
+        this.dispatchEvent(new CustomEvent("quicklook-overlay", { detail: obj }));
+        return obj;
+    }
+
+    private _arButton?: HTMLElement;
+    private async createQuicklookButton() {
+        if (!this.webxr) {
+            await delay(1);
+            this.webxr = GameObject.findObjectOfType(WebXR) ?? undefined;
+            if (this.webxr) {
+                if(this.webxr.VRButton) this.webxr.VRButton.parentElement?.removeChild(this.webxr.VRButton);
+                // check if we have an AR button already and re-use that
+                if (this.webxr.ARButton && this._arButton !== this.webxr.ARButton) {
+                    this._arButton = this.webxr.ARButton;
+                    // Hack to remove the immersiveweb link
+                    const linkInButton = this.webxr.ARButton.parentElement?.querySelector("a");
+                    if (linkInButton) {
+                        linkInButton.href = "";
+                    }
+                    this.webxr.ARButton.innerText = "Open in Quicklook";
+                    this.webxr.ARButton.disabled = false;
+                    this.webxr.ARButton.addEventListener("click", evt => {
+                        evt.preventDefault();
+                        this.exportAsync();
+                    });
+                    this.webxr.ARButton.classList.add("quicklook-ar-button");
+                }
+                // create a button if WebXR didnt create one yet
+                else {
+                    this.webxr.createARButton = false;
+                    this.webxr.createVRButton = false;
+                    let container = window.document.querySelector(".webxr-buttons");
+                    if (!container) {
+                        container = document.createElement("div");
+                        container.classList.add("webxr-buttons");
+                    }
+                    const button = document.createElement("button");
+                    button.innerText = "Open in Quicklook";
+                    button.addEventListener("click", () => {
+                        this.exportAsync();
+                    });
+                    button.classList.add('webxr-ar-button');
+                    button.classList.add('webxr-button');
+                    button.classList.add("quicklook-ar-button");
+                    container.appendChild(button);
+                }
+            }
+            else {
+                console.warn("Could not find WebXR component: will not create Quicklook button", Context.Current);
+            }
+        }
+    }
+
+    private resetStyles(el: HTMLElement) {
+        el.style.position = "";
+        el.style.top = "";
+        el.style.left = "";
+        el.style.width = "";
+        el.style.height = "";
+        el.style.margin = "";
+        el.style.padding = "";
+        el.style.border = "";
+        el.style.background = "";
+        el.style.color = "";
+        el.style.font = "";
+        el.style.textAlign = "";
+        el.style.opacity = "";
+        el.style.zIndex = "";
+    }
+}