@needle-tools/engine 2.52.0-pre → 2.53.0-pre

package/src/engine/extensions/{NEEDLE_deferred_texture.ts → NEEDLE_progressive.ts}

@@ -5,14 +5,13 @@ import { Context } from "../engine_setup";
 import { addDracoAndKTX2Loaders } from "../engine_loaders";
 import { getParam, getPath } from "../engine_utils";
 
-export const EXTENSION_NAME = "NEEDLE_deferred_texture";
+export const EXTENSION_NAME = "NEEDLE_progressive";
 
 const debug = getParam("debugprogressive");
 
-declare type DeferredTextureModel = {
+declare type ProgressiveTextureSchema = {
     uri: string;
     guid: string;
-    usage?: string,
 }
 
 const debug_toggle_maps: Map<Material, { [key: string]: { original: Texture, lod0: Texture } }> = new Map();
@@ -35,8 +34,7 @@ if (debug) {
     });
 }
 
-export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
-
+export class NEEDLE_progressive implements GLTFLoaderPlugin {
 
     static assignTextureLOD(context: Context, source: SourceIdentifier | undefined, material: Material, level: number = 0) {
         if (!material) return;
@@ -46,7 +44,7 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
 
                 if (debug) console.log("-----------\n", "FIND", material.name, slot, val?.name, val?.userData, val, material);
 
-                NEEDLE_deferred_texture.getOrLoadTexture(context, source, material, slot, val, level).then(t => {
+                NEEDLE_progressive.getOrLoadTexture(context, source, material, slot, val, level).then(t => {
                     if (t?.isTexture === true) {
 
                         if (debug) console.log("Assign LOD", material.name, slot, t.name, t["guid"], material, "Prev:", val, "Now:", t, "\n--------------");
@@ -109,14 +107,14 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
             console.log("AFTER", this.sourceId, gltf);
         this.parser.json.textures?.forEach((textureInfo, index) => {
             if (textureInfo?.extensions) {
-                const ext: DeferredTextureModel = textureInfo?.extensions[EXTENSION_NAME];
+                const ext: ProgressiveTextureSchema = textureInfo?.extensions[EXTENSION_NAME];
                 if (ext) {
                     const prom = this.parser.getDependency("texture", index);
                     this._loading.splice(this._loading.indexOf(index), 1);
                     prom.then(t => {
                         if (debug) console.log("register texture", t.name, t.uuid, ext);
                         t.userData.deferred = ext;
-                        NEEDLE_deferred_texture.cache.set(t.uuid, ext);
+                        NEEDLE_progressive.cache.set(t.uuid, ext);
                     });
                 }
             }
@@ -125,13 +123,13 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
         return null;
     }
 
-    private static cache = new Map<string, DeferredTextureModel>();
+    private static cache = new Map<string, ProgressiveTextureSchema>();
     private static resolved: { [key: string]: Texture } = {};
 
     private static async getOrLoadTexture(context: Context, source: SourceIdentifier | undefined, material: Material, slot: string, current: Texture, _level: number): Promise<Texture | null> {
 
         const key = current.uuid;
-        const ext: DeferredTextureModel | undefined = NEEDLE_deferred_texture.cache.get(key);// || current.userData.deferred;
+        const ext: ProgressiveTextureSchema | undefined = NEEDLE_progressive.cache.get(key);// || current.userData.deferred;
         if (ext) {
             if (debug)
                 console.log(key, ext.uri, ext.guid);
@@ -158,7 +156,7 @@ export class NEEDLE_deferred_texture implements GLTFLoaderPlugin {
                 for (const tex of gltf.parser.json.textures) {
                     index++;
                     if (tex?.extensions) {
-                        const other: DeferredTextureModel = tex?.extensions[EXTENSION_NAME];
+                        const other: ProgressiveTextureSchema = tex?.extensions[EXTENSION_NAME];
                         if (other?.guid) {
                             if (other.guid === ext.guid) {
                                 found = true;

package/src/engine/extensions/extensions.ts

@@ -12,7 +12,7 @@ import { SourceIdentifier } from "../engine_types";
 import { Context } from "../engine_setup";
 import { NEEDLE_lighting_settings } from "./NEEDLE_lighting_settings";
 import { NEEDLE_render_objects } from "./NEEDLE_render_objects";
-import { NEEDLE_deferred_texture } from "./NEEDLE_deferred_texture";
+import { NEEDLE_progressive } from "./NEEDLE_progressive";
 
 export function registerComponentExtension(loader: GLTFLoader): NEEDLE_components {
     const ext = new NEEDLE_components();
@@ -40,7 +40,7 @@ export function registerExtensions(loader: GLTFLoader, context: Context, sourceI
     loader.register(p => new NEEDLE_lighting_settings(p, sourceId, context));
     loader.register(p => new NEEDLE_techniques_webgl(p, sourceId));
     loader.register(p => new NEEDLE_render_objects(p, sourceId));
-    loader.register(p => new NEEDLE_deferred_texture(p, sourceId, context));
+    loader.register(p => new NEEDLE_progressive(p, sourceId, context));
     loader.register(p => new EXT_texture_exr(p));
 
     const setPointerResolverFunction = loader["setAnimationPointerResolver"];

package/src/engine-components/Camera.ts

@@ -9,6 +9,7 @@ import { XRSessionMode } from "../engine/engine_setup";
 import { ICamera } from "../engine/engine_types"
 import { showBalloonMessage } from "../engine/debug/debug";
 import { getWorldPosition } from "../engine/engine_three_utils";
+import { Gizmos } from "../engine/engine_gizmos";
 
 export enum ClearFlags {
     Skybox = 1,
@@ -17,6 +18,7 @@ export enum ClearFlags {
 }
 
 const debug = getParam("debugcam");
+const debugscreenpointtoray = getParam("debugscreenpointtoray");
 
 export class Camera extends Behaviour implements ICamera {
 
@@ -157,14 +159,15 @@ export class Camera extends Behaviour implements ICamera {
     private static _origin: THREE.Vector3 = new THREE.Vector3();
     private static _direction: THREE.Vector3 = new THREE.Vector3();
     public screenPointToRay(x: number, y: number, ray?: Ray): Ray {
+        let cam = this.cam;
         const origin = Camera._origin;
         origin.set(x, y, -1);
         this.context.input.convertScreenspaceToRaycastSpace(origin);
+        if(debugscreenpointtoray) console.log("screenPointToRay", x.toFixed(2), y.toFixed(2), "now:", origin.x.toFixed(2), origin.y.toFixed(2), "isInXR:" + this.context.isInXR);
         origin.z = -1;
-        origin.unproject(this.cam);
-
+        origin.unproject(cam);
         const dir = Camera._direction.set(origin.x, origin.y, origin.z);
-        const camPosition = getWorldPosition(this.cam);
+        const camPosition = getWorldPosition(cam);
         dir.sub(camPosition);
         dir.normalize();
         if (ray) {
@@ -180,6 +183,17 @@ export class Camera extends Behaviour implements ICamera {
         if (!this.sourceId) {
             console.warn("Camera has no source - the camera should be exported inside a gltf", this.name);
         }
+
+        if(debugscreenpointtoray){
+            window.addEventListener("pointerdown", evt => {
+                const px = evt.clientX;
+                const py = evt.clientY;
+                console.log("touch", px.toFixed(2), py.toFixed(2))
+                const ray = this.screenPointToRay(px, py);
+                const randomHex = "#" + Math.floor(Math.random()*16777215).toString(16);
+                Gizmos.DrawRay(ray.origin, ray.direction, randomHex, 10);
+            });
+        }
     }
 
     onEnable(): void {

package/src/engine-components/Light.ts

@@ -292,6 +292,7 @@ export class Light extends Behaviour implements ILight {
 
         if (lightAlreadyCreated && !this.light) {
             this.light = this.gameObject as unknown as THREE.Light;
+            this._intensity = this.light.intensity;
 
             switch (this.type) {
                 case LightType.Directional:
@@ -346,7 +347,8 @@ export class Light extends Behaviour implements ILight {
         if (this.light) {
             if (this._intensity >= 0)
                 this.light.intensity = this._intensity;
-            else this._intensity = this.light.intensity;
+            else  
+                this._intensity = this.light.intensity;
 
             if (this.shadows !== LightShadows.None) {
                 this.light.castShadow = true;

package/src/engine-components/OrbitControls.ts

@@ -11,6 +11,9 @@ import { getParam, isMobileDevice } from "../engine/engine_utils";
 
 const freeCam = getParam("freecam");
 
+const disabledKeys = { LEFT: "", UP: "", RIGHT: "", BOTTOM: "" };
+let defaultKeys: any = undefined;
+
 export class OrbitControls extends Behaviour {
     public get controls() {
         return this._controls;
@@ -73,6 +76,7 @@ export class OrbitControls extends Behaviour {
             if (cam)
                 this._cameraObject = cam;
             this._controls = new ThreeOrbitControls(cam!, this.context.renderer.domElement);
+            if (defaultKeys === undefined) defaultKeys = { ...this._controls.keys };
         }
 
         if (this._controls) {
@@ -80,12 +84,11 @@ export class OrbitControls extends Behaviour {
                 this.enablePan = true;
                 this.enableZoom = true;
                 this.middleClickToFocus = true;
-                if(isMobileDevice()) this.doubleClickToFocus = true;
+                if (isMobileDevice()) this.doubleClickToFocus = true;
             }
 
             this._controls.enableDamping = this.enableDamping;
-            //@ts-ignore (not in types)
-            this._controls.enableKeys = this.enableKeys;
+            this._controls.keys = this.enableKeys ? defaultKeys : disabledKeys;
             this._controls.autoRotate = this.autoRotate;
             this._controls.autoRotateSpeed = this.autoRotateSpeed;
             this._controls.enableZoom = this.enableZoom;
@@ -164,6 +167,7 @@ export class OrbitControls extends Behaviour {
             this._inputs += 1;
         }
         if (this._inputs > 0) {
+            this.autoRotate = false;
             this._controls.autoRotate = false;
             this._lerpCameraToTarget = false;
             this._lerpToTargetPosition = false;

package/src/engine-components/Renderer.ts

@@ -7,7 +7,7 @@ import { getParam } from "../engine/engine_utils";
 import { serializable } from "../engine/engine_serialization_decorator";
 import { AxesHelper, Material, Mesh, Object3D, SkinnedMesh, Texture, Vector4 } from "three";
 import { NEEDLE_render_objects } from "../engine/extensions/NEEDLE_render_objects";
-import { NEEDLE_deferred_texture } from "../engine/extensions/NEEDLE_deferred_texture";
+import { NEEDLE_progressive } from "../engine/extensions/NEEDLE_progressive";
 import { NEED_UPDATE_INSTANCE_KEY } from "../engine/engine_instancing";
 import { IRenderer, ISharedMaterials } from "../engine/engine_types";
 import { debug, ReflectionProbe } from "./ReflectionProbe";
@@ -465,11 +465,11 @@ export class Renderer extends Behaviour implements IRenderer {
             if (debugProgressiveLoading) {
                 console.log("Load material LOD (with delay)", material.name);
                 setTimeout(() => {
-                    NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
+                    NEEDLE_progressive.assignTextureLOD(this.context, this.sourceId, material);
                 }, 2000);
             }
             else {
-                NEEDLE_deferred_texture.assignTextureLOD(this.context, this.sourceId, material);
+                NEEDLE_progressive.assignTextureLOD(this.context, this.sourceId, material);
             }
         }
 

package/src/engine-components/Voip.ts

@@ -307,6 +307,11 @@ export class Voip extends Behaviour {
             return;
         }
 
+        if (utils.isiOS() && utils.isSafari()) {
+            console.log("VOIP is currently not supported on Safari iOS")
+            return;
+        }
+
         this.peer = new Peer();
         navigator["getUserMedia"] = (navigator["getUserMedia"] || navigator["webkitGetUserMedia"] || navigator["mozGetUserMedia"] || navigator["msGetUserMedia"]);
 
@@ -360,10 +365,15 @@ export class Voip extends Behaviour {
     onDisable(): void {
         console.log("TODO: close all");
         for (const key in this.currentIncomingCalls) {
-            const call = this.currentIncomingCalls[key];
-            call?.close();
-            const con = this.connections[key];
-            con?.close();
+            try {
+                const call = this.currentIncomingCalls[key];
+                call?.close();
+                const con = this.connections[key];
+                con?.close();
+            }
+            catch (err) {
+                console.error(err);
+            }
         }
     }
 

package/src/engine-components/codegen/components.ts

@@ -123,8 +123,6 @@ export { Avatar_Brain_LookAt } from "../avatar/Avatar_Brain_LookAt";
 export { Avatar_MouthShapes } from "../avatar/Avatar_MouthShapes";
 export { Avatar_MustacheShake } from "../avatar/Avatar_MustacheShake";
 export { LogStats } from "../debug/LogStats";
-export { GltfExportBox } from "../export/GltfExport";
-export { GltfExport } from "../export/GltfExport";
 export { RGBAColor } from "../js-extensions/RGBAColor";
 export { PlayableDirector } from "../timeline/PlayableDirector";
 export { SignalAsset } from "../timeline/SignalAsset";
@@ -160,3 +158,9 @@ export { Rect } from "../ui/RectTransform";
 export { RectTransform } from "../ui/RectTransform";
 export { SpatialHtml } from "../ui/SpatialHtml";
 export { Text } from "../ui/Text";
+export { GltfExportBox } from "../export/gltf/GltfExport";
+export { GltfExport } from "../export/gltf/GltfExport";
+export { USDZExporter } from "../export/usdz/USDZExporter";
+export { RegisteredAnimationInfo } from "../export/usdz/extensions/Animation";
+export { TransformData } from "../export/usdz/extensions/Animation";
+export { AnimationExtension } from "../export/usdz/extensions/Animation";

package/src/engine-components/export/{GltfExport.ts → gltf/GltfExport.ts}

@@ -1,12 +1,12 @@
-import { Behaviour, GameObject } from "../Component";
+import { Behaviour, GameObject } from "../../Component";
 import { GLTFExporter } from 'three/examples/jsm/exporters/GLTFExporter.js';
-import GLTFMeshGPUInstancingExtension from '../../include/three/EXT_mesh_gpu_instancing_exporter.js';
-import { Renderer } from "../Renderer";
+import GLTFMeshGPUInstancingExtension from '../../../include/three/EXT_mesh_gpu_instancing_exporter.js';
+import { Renderer } from "../../Renderer";
 import { Object3D, Vector3 } from "three";
-import { SerializationContext } from "../../engine/engine_serialization_core";
-import { NEEDLE_components } from "../../engine/extensions/NEEDLE_components";
-import { getWorldPosition } from "../../engine/engine_three_utils";
-import { BoxHelperComponent } from "../BoxHelperComponent";
+import { SerializationContext } from "../../../engine/engine_serialization_core";
+import { NEEDLE_components } from "../../../engine/extensions/NEEDLE_components";
+import { getWorldPosition } from "../../../engine/engine_three_utils";
+import { BoxHelperComponent } from "../../BoxHelperComponent";
 import { AnimationClip } from "three";
 
 

package/src/engine-components/export/usdz/Extension.ts

@@ -0,0 +1,12 @@
+import { USDZObject } from "./types";
+
+
+export interface IUSDZExporterExtension {
+
+    get extensionName(): string;
+    onBeforeBuildDocument?(context);
+    onAfterBuildDocument?(context);
+    onExportObject?(object, model : USDZObject, context);
+    onAfterSerialize?(context);
+    onAfterHierarchy?(context);
+}

package/src/engine-components/export/usdz/USDZExporter.ts

@@ -0,0 +1,216 @@
+import { delay, getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils";
+import { Object3D, Color } from "three";
+import * as THREE from "three";
+import { USDZExporter as ThreeUSDZExporter } from "three/examples/jsm/exporters/USDZExporter";
+import { AnimationExtension } from "./extensions/Animation"
+import { ensureQuicklookLinkIsCreated } from "./utils/quicklook";
+import { getFormattedDate } from "./utils/timeutils";
+import { registerAnimatorsImplictly } from "./utils/animationutils";
+import { IUSDZExporterExtension } from "./Extension";
+import { Behaviour, GameObject } from "../../Component";
+import { WebXR } from "../../WebXR"
+import { serializable } from "../../../engine/engine_serialization";
+import { showBalloonWarning } from "../../../engine/debug/debug";
+import { Context } from "../../../engine/engine_setup";
+
+const debug = getParam("debugusdz");
+
+export type QuickLookOverlay = {
+    callToAction?: string;
+    checkoutTitle?: string;
+    checkoutSubtitle?: string;
+}
+
+export class USDZExporter extends Behaviour {
+
+    @serializable(Object3D)
+    objectToExport?: THREE.Object3D;
+
+    @serializable()
+    autoExportAnimations: boolean = false;
+
+    extensions: IUSDZExporterExtension[] = [];
+
+    private link!: HTMLAnchorElement;
+    private webxr?: WebXR;
+
+
+    start() {
+        if (debug) {
+            window.addEventListener("keydown", (evt) => {
+                switch (evt.key) {
+                    case "t":
+                        this.exportAsync();
+                        break;
+                }
+            });
+            if (isMobileDevice()) {
+                setTimeout(() => {
+                    this.exportAsync();
+                }, 2000)
+            }
+        }
+        document.getElementById("open-in-ar")?.addEventListener("click", (evt) => {
+            evt.preventDefault();
+            this.exportAsync();
+        });
+
+        if (!this.objectToExport) this.objectToExport = this.gameObject;
+    }
+
+
+
+    onEnable() {
+        const ios = isiOS()
+        const safari = isSafari();
+        if (debug || (ios && safari)) {
+            this.createQuicklookButton();
+            this.lastCallback = this.quicklookCallback.bind(this);
+            this.link = ensureQuicklookLinkIsCreated(this.context);
+            this.link.addEventListener('message', this.lastCallback);
+        }
+    }
+
+    onDisable() {
+        this.link?.removeEventListener('message', this.lastCallback);
+    }
+
+    async exportAsync() {
+        if (!this.objectToExport) return;
+
+        const exporter = new ThreeUSDZExporter();
+        const extensions: any = [...this.extensions]
+
+        // collect animators and their clips
+        const animExt = new AnimationExtension();
+        extensions.push(animExt);
+
+        if (this.autoExportAnimations)
+            registerAnimatorsImplictly(this.objectToExport, animExt);
+
+        const eventArgs = { self: this, exporter: exporter, extensions: extensions };
+        this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
+
+        let name = "needle";
+        if (debug) name += "-" + getFormattedDate();
+
+        //@ts-ignore
+        exporter.debug = debug;
+        //@ts-ignore
+        const arraybuffer = await exporter.parse(this.objectToExport, extensions);
+        const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
+
+        // second file: USDA (without assets)
+        //@ts-ignore
+        // const usda = exporter.lastUsda;
+        // const blob2 = new Blob([usda], { type: 'text/plain' });
+        // this.link.download = name + ".usda";
+        // this.link.href = URL.createObjectURL(blob2);
+        // this.link.click();
+
+        // see https://developer.apple.com/documentation/arkit/adding_an_apple_pay_button_or_a_custom_action_in_ar_quick_look
+        const overlay = this.buildQuicklookOverlay();
+        const callToAction = overlay.callToAction ? encodeURIComponent(overlay.callToAction) : "";
+        const checkoutTitle = overlay.checkoutTitle ? encodeURIComponent(overlay.checkoutTitle) : "";
+        const checkoutSubtitle = overlay.checkoutSubtitle ? encodeURIComponent(overlay.checkoutSubtitle) : "";
+        this.link.href = URL.createObjectURL(blob) + `#callToAction=${callToAction}&checkoutTitle=${checkoutTitle}&checkoutSubtitle=${checkoutSubtitle}&`;
+
+
+        if (!this.lastCallback) {
+            this.lastCallback = this.quicklookCallback.bind(this);
+            this.link.addEventListener('message', this.lastCallback);
+        }
+
+        // open quicklook
+        this.link.download = name + ".usdz";
+        this.link.click();
+
+        // TODO detect QuickLook availability:
+        // https://webkit.org/blog/8421/viewing-augmented-reality-assets-in-safari-for-ios/#:~:text=inside%20the%20anchor.-,Feature%20Detection,-To%20detect%20support
+    }
+
+    private lastCallback?: any;
+
+    private quicklookCallback(event) {
+        if ((event as any)?.data == '_apple_ar_quicklook_button_tapped') {
+            if (debug) showBalloonWarning("Quicklook closed via call to action button");
+            this.dispatchEvent(new CustomEvent("quicklook-button-tapped", { detail: this }));
+        }
+    }
+
+    private buildQuicklookOverlay(): QuickLookOverlay {
+        const obj: QuickLookOverlay = {};
+        obj.callToAction = "Close";
+        obj.checkoutTitle = "🌵 Made with Needle";
+        obj.checkoutSubtitle = "_";
+        // Use the quicklook-overlay event to customize the overlay
+        this.dispatchEvent(new CustomEvent("quicklook-overlay", { detail: obj }));
+        return obj;
+    }
+
+    private _arButton?: HTMLElement;
+    private async createQuicklookButton() {
+        if (!this.webxr) {
+            await delay(1);
+            this.webxr = GameObject.findObjectOfType(WebXR) ?? undefined;
+            if (this.webxr) {
+                if(this.webxr.VRButton) this.webxr.VRButton.parentElement?.removeChild(this.webxr.VRButton);
+                // check if we have an AR button already and re-use that
+                if (this.webxr.ARButton && this._arButton !== this.webxr.ARButton) {
+                    this._arButton = this.webxr.ARButton;
+                    // Hack to remove the immersiveweb link
+                    const linkInButton = this.webxr.ARButton.parentElement?.querySelector("a");
+                    if (linkInButton) {
+                        linkInButton.href = "";
+                    }
+                    this.webxr.ARButton.innerText = "Open in Quicklook";
+                    this.webxr.ARButton.disabled = false;
+                    this.webxr.ARButton.addEventListener("click", evt => {
+                        evt.preventDefault();
+                        this.exportAsync();
+                    });
+                    this.webxr.ARButton.classList.add("quicklook-ar-button");
+                }
+                // create a button if WebXR didnt create one yet
+                else {
+                    this.webxr.createARButton = false;
+                    this.webxr.createVRButton = false;
+                    let container = window.document.querySelector(".webxr-buttons");
+                    if (!container) {
+                        container = document.createElement("div");
+                        container.classList.add("webxr-buttons");
+                    }
+                    const button = document.createElement("button");
+                    button.innerText = "Open in Quicklook";
+                    button.addEventListener("click", () => {
+                        this.exportAsync();
+                    });
+                    button.classList.add('webxr-ar-button');
+                    button.classList.add('webxr-button');
+                    button.classList.add("quicklook-ar-button");
+                    container.appendChild(button);
+                }
+            }
+            else {
+                console.warn("Could not find WebXR component: will not create Quicklook button", Context.Current);
+            }
+        }
+    }
+
+    private resetStyles(el: HTMLElement) {
+        el.style.position = "";
+        el.style.top = "";
+        el.style.left = "";
+        el.style.width = "";
+        el.style.height = "";
+        el.style.margin = "";
+        el.style.padding = "";
+        el.style.border = "";
+        el.style.background = "";
+        el.style.color = "";
+        el.style.font = "";
+        el.style.textAlign = "";
+        el.style.opacity = "";
+        el.style.zIndex = "";
+    }
+}