@needle-tools/engine 3.7.7-beta → 3.9.0-alpha

package/src/engine-components/RendererLightmap.ts

@@ -1,7 +1,9 @@
 import { Behaviour, GameObject } from "./Component";
-import * as THREE from "three";
-import { Texture } from "three";
+import { Material, Mesh, Shader, ShaderMaterial, Texture, Vector4 } from "three";
 import { Context, OnRenderCallback } from "../engine/engine_setup";
+import { getParam } from "../engine/engine_utils";
+
+const debug = getParam("debuglightmaps");
 
 // this component is automatically added by the Renderer if the object has lightmap uvs AND we have a lightmap
 // for multimaterial objects GLTF exports a "Group" with the renderer component
@@ -19,20 +21,20 @@ export class RendererLightmap {
     }
 
     lightmapIndex: number = -1;
-    lightmapScaleOffset: THREE.Vector4 = new THREE.Vector4(1, 1, 0, 0);
+    lightmapScaleOffset: THREE.Vector4 = new Vector4(1, 1, 0, 0);
 
     private context: Context;
-    private gameObject: GameObject;
-    private lightmapTexture: THREE.Texture | null = null;
-    private lightmapScaleOffsetUniform = { value: new THREE.Vector4(1, 1, 0, 0) };
-    private lightmapUniform: { value: THREE.Texture | null } = { value: null };
+    private gameObject: Mesh;
+    private lightmapTexture: Texture | null = null;
+    private lightmapScaleOffsetUniform = { value: new Vector4(1, 1, 0, 0) };
+    private lightmapUniform: { value: Texture | null } = { value: null };
 
-    constructor(gameObject: GameObject, context: Context) {
+    constructor(gameObject: Mesh, context: Context) {
         this.gameObject = gameObject;
         this.context = context;
     }
 
-    init(lightmapIndex: number, lightmapScaleOffset: THREE.Vector4, lightmapTexture: THREE.Texture, debug: boolean = false) {
+    init(lightmapIndex: number, lightmapScaleOffset: Vector4, lightmapTexture: Texture) {
         console.assert(this.gameObject !== undefined && this.gameObject !== null, "Missing gameobject", this);
 
         this.lightmapIndex = lightmapIndex;
@@ -40,24 +42,25 @@ export class RendererLightmap {
         this.lightmapScaleOffset = lightmapScaleOffset;
         this.lightmapTexture = lightmapTexture;
 
-        const debugLightmaps = debug;
-        if (debugLightmaps) this.setLightmapDebugMaterial();
+        if (debug) this.setLightmapDebugMaterial();
         this.applyLightmap();
     }
 
-    bindOnBeforeRender() {
-        this.context.removeBeforeRenderListener(this.gameObject, this.onBeforeRenderThreeComplete);
-        this.context.addBeforeRenderListener(this.gameObject, this.onBeforeRenderThreeComplete);
+    updateLightmapUniforms(material: Material) {
+        const uniforms = material["uniforms"];
+        if (uniforms && uniforms.lightmap) {
+            this.lightmapScaleOffsetUniform.value = this.lightmapScaleOffset;
+            this.lightmapUniform.value = this.lightmapTexture;
+            uniforms.lightmap = this.lightmapUniform;
+            uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
+        }
     }
 
-    private onBeforeRenderThreeComplete = (_renderer, _scene, _camera, _geometry, material, _group) => {
-        this.onBeforeRenderThree(material);
-    }
 
     private applyLightmap() {
-
         if (this.gameObject.type === "Object3D") {
-            // console.warn("Can not add lightmap. Is this object missing a renderer?");
+            if (debug)
+                console.warn("Can not add lightmap. Is this object missing a renderer?", this.gameObject.name);
             return;
         }
 
@@ -68,23 +71,28 @@ export class RendererLightmap {
 
         console.assert(this.gameObject.type === "Mesh", "Lightmap only works on meshes", this);
 
-        const mesh = this.gameObject as unknown as THREE.Mesh;
-        // TODO: ensure uv2 exists
-        if (!mesh.geometry.getAttribute("uv2"))
-            mesh.geometry.setAttribute("uv2", mesh.geometry.getAttribute("uv"));
-
-        const mat = this.gameObject["material"].clone();
-        this.gameObject["material"] = mat;
-
-        this.gameObject["material"].onBeforeCompile = (shader, _) => {
-            shader.uniforms.lightmap = this.lightmapUniform;
-            shader.uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
-        };
-
+        const mesh = this.gameObject as unknown as Mesh;
+        if (!mesh.geometry.getAttribute("uv1"))
+            mesh.geometry.setAttribute("uv1", mesh.geometry.getAttribute("uv"));
+
+        if (Array.isArray(this.gameObject.material)) {
+            const mats: Material[] = this.gameObject.material;
+            for (let i = 0; i < mats.length; i++) {
+                const mat = mats[i];
+                const cloned = mat.clone();
+                mats[i] = cloned;
+                cloned.onBeforeCompile = this.onBeforeCompile;
+            }
+        }
+        else {
+            const mat: Material = this.gameObject.material.clone();
+            this.gameObject.material = mat;
+            this.gameObject.material.onBeforeCompile = this.onBeforeCompile;
+        }
 
         if (this.lightmapIndex >= 0) {
             const lightmap = this.lightmapTexture;
-            const mat = this.gameObject["material"];
+            const mat = this.gameObject.material as any;
             if (mat && lightmap) {
                 if (!mat.uniforms) mat.uniforms = {};
                 mat.lightMap = lightmap;
@@ -93,27 +101,25 @@ export class RendererLightmap {
         }
     }
 
-    onBeforeRenderThree(material: THREE.Material) {
-
-        const uniforms = material["uniforms"];
-        if (uniforms && uniforms.lightmap) {
-            this.lightmapScaleOffsetUniform.value = this.lightmapScaleOffset;
-            this.lightmapUniform.value = this.lightmapTexture;
-            uniforms.lightmap = this.lightmapUniform;
-            uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
-        }
+    private onBeforeCompile = (shader: Shader, _) => {
+        if(debug) console.log("Lightmaps, before compile")
+        //@ts-ignore
+        shader.lightMapUv = "uv1";
+        this.lightmapScaleOffsetUniform.value = this.lightmapScaleOffset;
+        this.lightmapUniform.value = this.lightmapTexture;
+        shader.uniforms.lightmap = this.lightmapUniform;
+        shader.uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
     }
 
     private setLightmapDebugMaterial() {
 
         // debug lightmaps
-        this.gameObject["material"] = new THREE.ShaderMaterial({
+        this.gameObject["material"] = new ShaderMaterial({
             vertexShader: `
-                attribute vec2 uv2;
-                varying vec2 vUv2;
+                varying vec2 vUv1;
                 void main()
                 {
-                    vUv2 = uv2;
+                    vUv1 = uv1;
                     gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
                 }
                 `,
@@ -121,7 +127,7 @@ export class RendererLightmap {
                 uniform sampler2D lightmap;
                 uniform float lightMapIntensity;
                 uniform vec4 lightmapScaleOffset;
-                varying vec2 vUv2;
+                varying vec2 vUv1;
 
                 // took from threejs 05fc79cd52b79e8c3e8dec1e7dca72c5c39983a4
                 vec4 conv_sRGBToLinear( in vec4 value ) {
@@ -129,7 +135,7 @@ export class RendererLightmap {
                 }
 
                 void main() {
-                    vec2 lUv = vUv2.xy * lightmapScaleOffset.xy + vec2(lightmapScaleOffset.z, (1. - (lightmapScaleOffset.y + lightmapScaleOffset.w)));
+                    vec2 lUv = vUv1.xy * lightmapScaleOffset.xy + vec2(lightmapScaleOffset.z, (1. - (lightmapScaleOffset.y + lightmapScaleOffset.w)));
                     
                     vec4 lightMapTexel = texture2D( lightmap, lUv);
                     // The range of RGBM lightmaps goes from 0 to 34.49 (5^2.2) in linear space, and from 0 to 5 in gamma space.
@@ -138,7 +144,7 @@ export class RendererLightmap {
                     //lightMapTexel = conv_sRGBToLinear(lightMapTexel);
                     // lightMapTexel.rgb = vec3(1.);
 
-                    // gl_FragColor = vec4(vUv2.xy, 0, 1);
+                    // gl_FragColor = vec4(vUv1.xy, 0, 1);
                     gl_FragColor = lightMapTexel;
                 }
                 `,

package/src/engine-components/codegen/components.ts

@@ -184,6 +184,7 @@ export { UIRootComponent } from "../ui/BaseUIComponent";
 export { UsageMarker } from "../Interactable";
 export { USDZExporter } from "../export/usdz/USDZExporter";
 export { USDZText } from "../export/usdz/extensions/USDZText";
+export { USDZUIExtension } from "../export/usdz/extensions/USDZUI";
 export { VariantAction } from "../export/usdz/extensions/behavior/Actions";
 export { VelocityOverLifetimeModule } from "../ParticleSystemModules";
 export { VerticalLayoutGroup } from "../ui/Layout";

package/src/engine-components/export/usdz/ThreeUSDZExporter.ts

@@ -657,7 +657,7 @@ function addResources( object, context: USDZExporterContext ) {
 
 	if ( geometry ) {
 
-		if ( material.isMeshStandardMaterial ) { // || material.isMeshBasicMaterial // TODO convert unlit to lit+emissive
+		if ( material.isMeshStandardMaterial || material.isMeshBasicMaterial )  { // TODO convert unlit to lit+emissive
 
 			const geometryFileName = 'geometries/Geometry_' + geometry.id + '.usd';
 
@@ -1124,7 +1124,7 @@ ${array.join( '' )}
 
 }
 
-function buildMaterial( material: MeshStandardMaterial, textures, quickLookCompatible = false ) {
+function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatible = false ) {
 
 	// https://graphics.pixar.com/usd/docs/UsdPreviewSurface-Proposal.html
 
@@ -1182,7 +1182,9 @@ function buildMaterial( material: MeshStandardMaterial, textures, quickLookCompa
 
 		const needsTextureScale = mapType !== 'normal' && (color && (color.r !== 1 || color.g !== 1 || color.b !== 1 || opacity !== 1)) || false;
 		const needsNormalScaleAndBias = mapType === 'normal';
-		const normalScaleValueString = (material.normalScale ? material.normalScale.x * 2 : 2).toFixed( PRECISION );
+		const normalScaleValueString = material instanceof MeshStandardMaterial
+			? (material.normalScale ? material.normalScale.x * 2 : 2).toFixed( PRECISION )
+			: "1";
 
 		return `
         ${needsTextureTransform ? `def Shader "Transform2d_${mapType}" (
@@ -1254,72 +1256,76 @@ function buildMaterial( material: MeshStandardMaterial, textures, quickLookCompa
 
 	}
 
-	if ( material.emissiveMap ) {
+	if ( material.aoMap ) {
 
-		inputs.push( `${pad}color3f inputs:emissiveColor.connect = </Materials/Material_${material.id}/Texture_${material.emissiveMap.id}_emissive.outputs:rgb>` );
+		inputs.push( `${pad}float inputs:occlusion.connect = </Materials/Material_${material.id}/Texture_${material.aoMap.id}_occlusion.outputs:r>` );
 
-		samplers.push( buildTexture( material.emissiveMap, 'emissive' ) );
+		samplers.push( buildTexture( material.aoMap, 'occlusion' ) );
+
+	}
 
-	} else if ( material.emissive?.getHex() > 0 ) {
+	if ( material.alphaMap ) {
+
+		inputs.push( `${pad}float inputs:opacity.connect = </Materials/Material_${material.id}/Texture_${material.alphaMap.id}_opacity.outputs:r>` );
+		inputs.push( `${pad}float inputs:opacityThreshold = 0.0001` );
 
-		inputs.push( `${pad}color3f inputs:emissiveColor = ${buildColor( material.emissive )}` );
+		samplers.push( buildTexture( material.alphaMap, 'opacity' ) );
 
 	} else {
-		
-		inputs.push( `${pad}color3f inputs:emissiveColor = (0, 0, 0)` );
+
+		inputs.push( `${pad}float inputs:opacity = ${effectiveOpacity}` );
 
 	}
 
-	if ( material.normalMap ) {
+	if ( material instanceof MeshStandardMaterial ) { 
 
-		inputs.push( `${pad}normal3f inputs:normal.connect = </Materials/Material_${material.id}/Texture_${material.normalMap.id}_normal.outputs:rgb>` );
+		if ( material.emissiveMap ) {
 
-		samplers.push( buildTexture( material.normalMap, 'normal' ) );
+			inputs.push( `${pad}color3f inputs:emissiveColor.connect = </Materials/Material_${material.id}/Texture_${material.emissiveMap.id}_emissive.outputs:rgb>` );
 
-	}
+			samplers.push( buildTexture( material.emissiveMap, 'emissive' ) );
 
-	if ( material.aoMap ) {
-
-		inputs.push( `${pad}float inputs:occlusion.connect = </Materials/Material_${material.id}/Texture_${material.aoMap.id}_occlusion.outputs:r>` );
+		} else if ( material.emissive?.getHex() > 0 ) {
 
-		samplers.push( buildTexture( material.aoMap, 'occlusion' ) );
+			inputs.push( `${pad}color3f inputs:emissiveColor = ${buildColor( material.emissive )}` );
 
-	}
+		} else {
+			
+			inputs.push( `${pad}color3f inputs:emissiveColor = (0, 0, 0)` );
 
-	if ( material.roughnessMap && material.roughness === 1 ) {
+		}
 
-		inputs.push( `${pad}float inputs:roughness.connect = </Materials/Material_${material.id}/Texture_${material.roughnessMap.id}_roughness.outputs:g>` );
+		if ( material.normalMap ) {
 
-		samplers.push( buildTexture( material.roughnessMap, 'roughness' ) );
+			inputs.push( `${pad}normal3f inputs:normal.connect = </Materials/Material_${material.id}/Texture_${material.normalMap.id}_normal.outputs:rgb>` );
 
-	} else {
+			samplers.push( buildTexture( material.normalMap, 'normal' ) );
 
-		inputs.push( `${pad}float inputs:roughness = ${material.roughness !== undefined ? material.roughness : 1 }` );
+		}
 
-	}
+		if ( material.roughnessMap && material.roughness === 1 ) {
 
-	if ( material.metalnessMap && material.metalness === 1 ) {
+			inputs.push( `${pad}float inputs:roughness.connect = </Materials/Material_${material.id}/Texture_${material.roughnessMap.id}_roughness.outputs:g>` );
 
-		inputs.push( `${pad}float inputs:metallic.connect = </Materials/Material_${material.id}/Texture_${material.metalnessMap.id}_metallic.outputs:b>` );
+			samplers.push( buildTexture( material.roughnessMap, 'roughness' ) );
 
-		samplers.push( buildTexture( material.metalnessMap, 'metallic' ) );
+		} else {
 
-	} else {
+			inputs.push( `${pad}float inputs:roughness = ${material.roughness !== undefined ? material.roughness : 1 }` );
 
-		inputs.push( `${pad}float inputs:metallic = ${material.metalness !== undefined ? material.metalness : 0 }` );
+		}
 
-	}
+		if ( material.metalnessMap && material.metalness === 1 ) {
 
-	if ( material.alphaMap ) {
+			inputs.push( `${pad}float inputs:metallic.connect = </Materials/Material_${material.id}/Texture_${material.metalnessMap.id}_metallic.outputs:b>` );
 
-		inputs.push( `${pad}float inputs:opacity.connect = </Materials/Material_${material.id}/Texture_${material.alphaMap.id}_opacity.outputs:r>` );
-		inputs.push( `${pad}float inputs:opacityThreshold = 0.0001` );
+			samplers.push( buildTexture( material.metalnessMap, 'metallic' ) );
 
-		samplers.push( buildTexture( material.alphaMap, 'opacity' ) );
+		} else {
 
-	} else {
+			inputs.push( `${pad}float inputs:metallic = ${material.metalness !== undefined ? material.metalness : 0 }` );
 
-		inputs.push( `${pad}float inputs:opacity = ${effectiveOpacity}` );
+		}
 
 	}
 

package/src/engine-components/export/usdz/USDZExporter.ts

@@ -16,6 +16,7 @@ import { hasProLicense } from "../../../engine/engine_license";
 import { BehaviorExtension } from "./extensions/behavior/Behaviour";
 import { AudioExtension } from "./extensions/behavior/AudioExtension";
 import { TextExtension } from "./extensions/USDZText";
+import { USDZUIExtension } from "./extensions/USDZUI";
 import { Renderer } from "../../Renderer"
 
 const debug = getParam("debugusdz");
@@ -97,6 +98,7 @@ export class USDZExporter extends Behaviour {
             this.extensions.push(new BehaviorExtension());
             this.extensions.push(new AudioExtension());
             this.extensions.push(new TextExtension());
+            this.extensions.push(new USDZUIExtension());
         }
     }
 

package/src/engine-components/export/usdz/extensions/USDZText.ts

@@ -146,8 +146,13 @@ export class TextExtension implements IUSDExporterExtension {
         return "text";
     }
 
-    onExportObject(object: Object3D, model: USDObject, _context: USDZExporterContext) {
+    // HACK we should clean this up, text export has moved to USDZUI.ts and is
+    // integrated into the hierarchy now
+    onExportObject(_object: Object3D, _model: USDObject, _context: USDZExporterContext) {
 
+        return;
+
+        /*
         const text = GameObject.getComponent(object, Text);
         if (text) {
             const rt = GameObject.getComponent(object, RectTransform);
@@ -186,6 +191,41 @@ export class TextExtension implements IUSDExporterExtension {
                 textObj.writeTo(undefined, writer);
             });
         }
+        */
+    }
+
+    exportText(object: Object3D, newModel: USDObject, _context: USDZExporterContext) {
+
+        const text = GameObject.getComponent(object, Text);
+        if (!text) return;
+
+        const rt = GameObject.getComponent(object, RectTransform);
+        let width = 100;
+        let height = 100;
+        if (rt) {
+            width = rt.width;
+            height = rt.height;
+        }
+
+        newModel.matrix = rotateYAxisMatrix.clone();
+        if (rt) // Not ideal but works for now:
+            newModel.matrix.premultiply(invertX);
+
+        const color = new Color().copySRGBToLinear(text.color);
+        newModel.material = new MeshStandardMaterial({ color: color, emissive: color });
+
+        newModel.addEventListener("serialize", (writer: USDWriter, _context: USDZExporterContext) => {
+            let txt = text.text;
+            txt = txt.replace(/\n/g, "\\n");
+            const textObj = TextBuilder.multiLine(txt, width, height, HorizontalAlignment.center, VerticalAlignment.bottom, TextWrapMode.flowing);
+            this.setTextAlignment(textObj, text.alignment);
+            this.setOverflow(textObj, text);
+            if (newModel.material)
+                textObj.material = newModel.material;
+            textObj.pointSize = this.convertToTextSize(text.fontSize);
+            textObj.depth = .001;
+            textObj.writeTo(undefined, writer);
+        });
     }
 
     private convertToTextSize(pixel: number) {

package/src/engine-components/export/usdz/extensions/USDZUI.ts

@@ -0,0 +1,120 @@
+import { IUSDExporterExtension } from "../Extension";
+import { USDObject, USDZExporterContext } from "../ThreeUSDZExporter";
+import { GameObject } from "../../../Component";
+import { Canvas } from "../../../ui/Canvas";
+import { CanvasGroup } from "../../../ui/CanvasGroup";
+import { $shadowDomOwner } from "../../../ui/BaseUIComponent";
+import { RectTransform } from "../../../ui/RectTransform";
+import { Color, Mesh, MeshBasicMaterial, Object3D } from "three";
+import { TextExtension } from "./USDZText";
+import { RenderMode } from "../../../ui/Canvas";
+
+export class USDZUIExtension implements IUSDExporterExtension {
+    get extensionName(): string {
+        return "tmui";
+    }
+
+    // TODO would probably be better to export each object instead of the entire Canvas
+    // so that we don't export them twice (once as regular hierarchy, once as part of Canvas export)
+    onExportObject(object: Object3D, model: USDObject, _context: USDZExporterContext) {
+        const canvas = GameObject.getComponent(object, Canvas);
+
+        if (canvas && canvas.activeAndEnabled && canvas.renderMode === RenderMode.WorldSpace) {
+            const textExt = new TextExtension();
+            const rt = GameObject.getComponent(object, RectTransform);
+            const canvasGroup = GameObject.getComponent(object, CanvasGroup);
+
+            let width = 100;
+            let height = 100;
+            if (rt) {
+                width = rt.width;
+                height = rt.height;
+
+                const shadowRootModel = USDObject.createEmpty();
+                const shadowComponent = rt.shadowComponent;
+                model.add(shadowRootModel);
+
+                if (shadowComponent) {
+                    const mat = shadowComponent.matrix;
+                    shadowRootModel.matrix.copy(mat);
+                    // shadowRootModel.matrix.premultiply(invertX);
+
+                    // TODO build map of parent GOs to USDObjects so we can reparent while traversing
+                    const usdObjectMap = new Map<Object3D, USDObject>();
+                    const opacityMap = new Map<Object3D, number>();
+                    usdObjectMap.set(shadowComponent, shadowRootModel);
+                    opacityMap.set(shadowComponent, canvasGroup ? canvasGroup.alpha : 1);
+
+                    shadowComponent.traverse((child) => {
+                        if (child === shadowComponent) return;
+
+                        const childModel = USDObject.createEmpty();
+                        childModel.matrix.copy(child.matrix);
+
+                        const childParent = child.parent;
+                        const isText = childParent && typeof childParent["textContent"] === "string" && childParent["textContent"].length;
+                        let hierarchyOpacity = opacityMap.get(childParent!) || 1;
+                        
+                        // TODO CanvasGroup doesn't render something but modifies opacity
+                        // get CanvasGroup and modify alpha here
+
+                        const canvasGroup = GameObject.getComponent(child, CanvasGroup);
+                        if (canvasGroup)
+                            hierarchyOpacity *= canvasGroup.alpha;
+
+                        if (child instanceof Mesh && isText) {
+                            // get shadoDomOwner so we can export Text from the text extension directly
+                            const shadowDomOwner = child[$shadowDomOwner].gameObject;
+                            textExt.exportText(shadowDomOwner, childModel, _context);
+                        }
+
+                        if (child instanceof Mesh && !isText)
+                        {
+                            // UI behaves weird: it seems it's always flipped right now,
+                            // and three magically fixes it when rendering
+                            // see https://github.com/mrdoob/three.js/pull/12720#issue-275923930
+                            // https://github.com/mrdoob/three.js/issues/17361
+                            // So we need to fix the winding order after applying the negative scale.
+                            const clonedGeo = child.geometry.clone();
+                            clonedGeo.scale(1, 1, -1);
+                            this.flipWindingOrder(clonedGeo);
+                            childModel.geometry = clonedGeo;
+
+                            const color = new Color();
+                            const ownOpacity = child.material.opacity;
+                            color.copy(child.material.color);
+
+                            // Calculate opacity from parent chain and own alpha
+                            childModel.material = new MeshBasicMaterial({
+                                color: color,
+                                opacity: ownOpacity * hierarchyOpacity,
+                                map: child.material.map,
+                                transparent: true,
+                            });
+                        }
+
+                        usdObjectMap.set(child, childModel);
+                        opacityMap.set(child, hierarchyOpacity);
+
+                        const parentUsdzObject = usdObjectMap.get(childParent!);
+                        if (!parentUsdzObject) {
+                            console.error("Error when exporting UI: shadow component parent not found!", child, child.parent);
+                            return;
+                        }
+                        parentUsdzObject.add(childModel);
+                    });
+                }
+            }
+        }
+    }
+    
+    private flipWindingOrder(geometry) {
+        const index = geometry.index.array
+        for (let i = 0, il = index.length / 3; i < il; i++) {
+          let x = index[i * 3]
+          index[i * 3] = index[i * 3 + 2]
+          index[i * 3 + 2] = x
+        }
+        geometry.index.needsUpdate = true
+    }
+}

package/src/engine-components/ui/BaseUIComponent.ts

@@ -6,8 +6,11 @@ import { showGizmos } from '../../engine/engine_default_parameters';
 import { AxesHelper, Object3D } from 'three';
 import { ICanvas, IGraphic } from './Interfaces';
 import { ShadowCastingMode } from '../Renderer';
+import { getParam } from '../../engine/engine_utils';
 export const includesDir = "./include";
 
+const debug = getParam("debugshadowcomponents");
+
 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
 
 
@@ -128,6 +131,8 @@ export class BaseUIComponent extends Behaviour {
         // otherwise it will fail when object are enabled at runtime
         if (needsUpdate)
             ThreeMeshUI.update();
+
+        if(debug) console.log(this.shadowComponent)
     }
 
 

package/src/engine-components/ui/Button.ts

@@ -116,8 +116,10 @@ export class Button extends Behaviour implements IPointerEventHandler {
         }
     }
 
-    onPointerClick(_args: PointerEventData) {
+    onPointerClick(args: PointerEventData) {
         if (!this.interactable) return;
+        // Button clicks should only run with left mouse button
+        if(args.pointerId !== 0) return;
         if (debug) {
             console.warn("Button Click", this.onClick);
             showBalloonMessage("CLICKED button " + this.name + " at " + this.context.time.frameCount);

package/src/engine-components/ui/Canvas.ts

@@ -190,6 +190,7 @@ export class Canvas extends UIRootComponent implements ICanvas {
     }
 
     onBeforeRenderRoutine = () => {
+        if(this.context.isInVR) return;
         this.previousParent = this.gameObject.parent;
         // console.log(this.previousParent?.name + "/" + this.gameObject.name);
 
@@ -208,6 +209,7 @@ export class Canvas extends UIRootComponent implements ICanvas {
     }
 
     onAfterRenderRoutine = () => {
+        if(this.context.isInVR) return;
         if ((this.screenspace || this.renderOnTop) && this.previousParent && this.context.mainCamera) {
             if (this.screenspace) {
                 const camObj = this.context.mainCamera;
@@ -215,7 +217,10 @@ export class Canvas extends UIRootComponent implements ICanvas {
             } else {
                 this.previousParent.add(this.gameObject);
             }
+            const prevAutoClearDepth = this.context.renderer.autoClear;
+            const prevAutoClearColor = this.context.renderer.autoClearColor;
             this.context.renderer.autoClear = false;
+            this.context.renderer.autoClearColor = false;
             this.context.renderer.clearDepth();
             this.onUpdateRenderMode(true);
             this.handleLayoutUpdates();
@@ -223,7 +228,8 @@ export class Canvas extends UIRootComponent implements ICanvas {
             // this.handleLayoutUpdates();
             EventSystem.ensureUpdateMeshUI(ThreeMeshUI, this.context);
             this.context.renderer.render(this.gameObject, this.context.mainCamera);
-            this.context.renderer.autoClear = true;
+            this.context.renderer.autoClear = prevAutoClearDepth;
+            this.context.renderer.autoClearColor = prevAutoClearColor;
             this.previousParent.add(this.gameObject);
         }
         this._lastMatrixWorld?.copy(this.gameObject.matrixWorld);

package/src/engine-components/ui/EventSystem.ts

@@ -127,6 +127,7 @@ export class EventSystem extends Behaviour {
             MeshUIHelper.resetLastSelected();
             const opts = new PointerEventData(this.context.input);
             opts.inputSource = ctrl;
+            opts.pointerId = 0;
             opts.isDown = ctrl.selectionDown;
             opts.isUp = ctrl.selectionUp;
             opts.isPressed = ctrl.selectionPressed;
@@ -136,10 +137,11 @@ export class EventSystem extends Behaviour {
                 args.grab = null;
             };
         }
-        this._selectEndFn ??= (ctrl, args: { grab: THREE.Object3D }) => {
+        this._selectEndFn ??= (ctrl: WebXRController, args: { grab: THREE.Object3D }) => {
             if (!args.grab) return;
             const opts = new PointerEventData(this.context.input);
             opts.inputSource = ctrl;
+            opts.pointerId = 0;
             opts.isDown = ctrl.selectionDown;
             opts.isUp = ctrl.selectionUp;
             opts.isPressed = ctrl.selectionPressed;

package/src/engine-components/ui/Graphic.ts

@@ -3,7 +3,7 @@ import * as ThreeMeshUI from 'three-mesh-ui'
 import { RGBAColor } from "../js-extensions/RGBAColor"
 import { BaseUIComponent } from "./BaseUIComponent";
 import { serializable } from '../../engine/engine_serialization_decorator';
-import { Color, LinearEncoding, sRGBEncoding, Texture } from 'three';
+import { Color, LinearSRGBColorSpace, SRGBColorSpace, Texture } from 'three';
 import { RectTransform } from './RectTransform';
 import { onChange, scheduleAction } from "./Utils"
 import { GameObject } from '../Component';
@@ -191,12 +191,12 @@ export class Graphic extends BaseUIComponent implements IGraphic, IRectTransform
         this.setOptions({ backgroundOpacity: 0 });
         if (tex) {
             // workaround for https://github.com/needle-tools/needle-engine-support/issues/109
-            if (tex.encoding === sRGBEncoding) {
+            if (tex.colorSpace === SRGBColorSpace) {
                 if (Graphic.textureCache.has(tex)) {
                     tex = Graphic.textureCache.get(tex)!;
                 } else {
                     const clone = tex.clone();
-                    clone.encoding = LinearEncoding;
+                    clone.colorSpace = LinearSRGBColorSpace;
                     Graphic.textureCache.set(tex, clone);
                     tex = clone;
                 }