@needle-tools/engine 4.13.1 → 4.14.0-next.31f837e

package/src/engine-components/RendererLightmap.ts

@@ -1,22 +1,19 @@
 import { NEEDLE_progressive } from "@needle-tools/gltf-progressive";
-import { Group, Material, Mesh, MeshPhysicalMaterial, Object3D, ShaderMaterial, Texture, Vector4, WebGLProgramParametersWithUniforms } from "three";
+import { Group, Mesh, Object3D, ShaderMaterial, Texture, Vector2, Vector4 } from "three";
 
 import type { Context } from "../engine/engine_setup.js";
 import { getParam } from "../engine/engine_utils.js";
+import { MaterialPropertyBlock } from "../engine/engine_materialpropertyblock.js";
 import { type Renderer } from "./Renderer.js";
 
 const debug = getParam("debuglightmaps");
 
-declare type MaterialWithLightmap = Material & { lightMap?: Texture | null };
-
-let cloningCounter = 0;
-
-const $lightmapVersion = Symbol("lightmap-material-version");
+const $lightmapKey = Symbol("lightmapKey");
 
 
 /**
  * This component is automatically added by the {@link Renderer} component if the object has lightmap uvs AND we have a lightmap.
- * 
+ *
  * @category Rendering
  * @group Components
  */
@@ -29,9 +26,13 @@ export class RendererLightmap {
         if (tex !== this.lightmapTexture) {
             this.lightmapTexture = tex;
             this.applyLightmap();
+            this.updatePropertyBlockTexture();
             if (this.lightmapTexture) {
                 NEEDLE_progressive.assignTextureLOD(this.lightmapTexture, 0).then(res => {
-                    if ((res as Texture)?.isTexture) this.lightmapTexture = res as Texture;
+                    if ((res as Texture)?.isTexture) {
+                        this.lightmapTexture = res as Texture;
+                        this.updatePropertyBlockTexture();
+                    }
                 })
             }
         }
@@ -41,13 +42,12 @@ export class RendererLightmap {
     private lightmapScaleOffset: Vector4 = new Vector4(1, 1, 0, 0);
 
     private readonly renderer: Renderer;
-    private readonly clonedMaterials = new Array<Material>();
+    private _isApplied: boolean = false;
 
     private get context(): Context { return this.renderer.context; }
     private get gameObject() { return this.renderer.gameObject; }
+
     private lightmapTexture: Texture | null = null;
-    private lightmapScaleOffsetUniform = { value: new Vector4(1, 1, 0, 0) };
-    private lightmapUniform: { value: Texture | null } = { value: null };
 
     constructor(renderer: Renderer) {
         this.renderer = renderer;
@@ -60,7 +60,10 @@ export class RendererLightmap {
         this.lightmapScaleOffset = lightmapScaleOffset;
         this.lightmapTexture = lightmapTexture;
         NEEDLE_progressive.assignTextureLOD(lightmapTexture, 0).then(res => {
-            if ((res as Texture)?.isTexture) this.lightmapTexture = res as Texture;
+            if ((res as Texture)?.isTexture) {
+                this.lightmapTexture = res as Texture;
+                this.updatePropertyBlockTexture();
+            }
         })
         if (debug == "show") {
             console.log("Lightmap:", this.gameObject.name, lightmapIndex, "\nScaleOffset:", lightmapScaleOffset, "\nTexture:", lightmapTexture)
@@ -70,18 +73,18 @@ export class RendererLightmap {
         this.applyLightmap();
     }
 
-    updateLightmapUniforms(material: Material) {
-        const uniforms = material["uniforms"];
-        if (uniforms && uniforms.lightmap) {
-            this.lightmapScaleOffsetUniform.value = this.lightmapScaleOffset;
-            uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
-        }
+    updateLightmapUniforms(_material: any) {
     }
 
     /**
-     * Apply the lightmap to the object. This will clone the material and set the lightmap texture and scale/offset
+     * Apply the lightmap to the object using MaterialPropertyBlock instead of cloning materials.
+     * The lightmap texture and its per-object UV transform are set as overrides via PropertyBlock.
+     * Three.js reads material.lightMap to determine shader defines and upload uniforms,
+     * and uses texture.offset/repeat to compute lightMapTransform in the vertex shader.
      */
     applyLightmap() {
+        if (this._isApplied) return;
+
         if (this.gameObject.type === "Object3D") {
             if (debug)
                 console.warn("Can not add lightmap. Is this object missing a renderer?", this.gameObject.name);
@@ -91,23 +94,61 @@ export class RendererLightmap {
         const mesh = this.gameObject as unknown as (Mesh | Group);
         this.ensureLightmapUvs(mesh);
 
-        for (let i = 0; i < this.renderer.sharedMaterials.length; i++) {
-
-            const mat = this.renderer.sharedMaterials[i];
-            if (!mat) continue;
-            const newMat = this.ensureLightmapMaterial(mat, i);
-            if (mat !== newMat) {
-                this.renderer.sharedMaterials[i] = newMat;
+        if (this.lightmapIndex >= 0 && this.lightmapTexture) {
+            this.lightmapTexture.channel = 1;
+            const so = this.lightmapScaleOffset;
+            for (let i = 0; i < this.renderer.sharedMaterials.length; i++) {
+                const mat = this.renderer.sharedMaterials[i];
+                if (!mat) continue;
+
+                // If the material doesn't support lightmaps, skip it
+                if (mat["lightMap"] === undefined) continue;
+
+                if (debug) console.log("Setting lightmap on material", mat.name, "for renderer", this.renderer.name);
+
+                // Use property block to set lightMap with per-object UV transform.
+                // The texture transform differentiates cache keys per object and
+                // PropertyBlock handles save/restore of the shared texture's offset/repeat.
+                const propertyBlock = MaterialPropertyBlock.get(this.gameObject);
+                propertyBlock.setOverride("lightMap", this.lightmapTexture, {
+                    offset: new Vector2(so.z, 1 - so.y - so.w),
+                    repeat: new Vector2(so.x, so.y)
+                });
+
+                (mat as any)[$lightmapKey] = true;
             }
+
+            this._isApplied = true;
         }
+    }
 
-        if (this.lightmapIndex >= 0 && this.lightmapTexture) {
-            // always on channel 1 for now. We could optimize this by passing the correct lightmap index along
-            this.lightmapTexture.channel = 1;
-            for (const mat of this.renderer.sharedMaterials) {
-                if (mat) this.assignLightmapTexture(mat);
+    /** Update the lightMap override on all property blocks (e.g. after LOD swap) */
+    private updatePropertyBlockTexture() {
+        if (!this._isApplied || !this.lightmapTexture) return;
+        this.lightmapTexture.channel = 1;
+        const so = this.lightmapScaleOffset;
+        const propertyBlock = MaterialPropertyBlock.get(this.gameObject);
+        propertyBlock.setOverride("lightMap", this.lightmapTexture, {
+            offset: new Vector2(so.z, 1 - so.y - so.w),
+            repeat: new Vector2(so.x, so.y)
+        });
+    }
+
+    /**
+     * Remove the lightmap from the object
+     */
+    onUnset() {
+        for (let i = 0; i < this.renderer.sharedMaterials.length; i++) {
+            const mat = this.renderer.sharedMaterials[i];
+            if (mat) {
+                delete (mat as any)[$lightmapKey];
             }
         }
+        const block = MaterialPropertyBlock.get(this.gameObject);
+        if (block) {
+            block.clearOverride("lightMap");
+            block.clearOverride("lightMapIntensity");
+        }
     }
 
     private ensureLightmapUvs(object: Object3D | Group | Mesh) {
@@ -123,59 +164,8 @@ export class RendererLightmap {
         }
     }
 
-    private ensureLightmapMaterial(material: Material, index: number) {
-        if (!material.userData) material.userData = {};
-        // if (material instanceof MeshPhysicalMaterial) {
-        //     return material;
-        // }
-        // check if the material version has changed and only then clone the material
-        if (this.clonedMaterials[index] !== material) {
-            if (debug) {
-                ++cloningCounter;
-                if (cloningCounter++ < 1000) {
-                    console.warn(`Cloning material for lightmap ${this.renderer.name}: '${material.name}'`);
-                }
-                else if (cloningCounter === 1000) {
-                    console.warn(`Further material cloning for lightmaps suppressed to avoid flooding the console.`);
-                }
-            }
-            const mat: Material = material.clone();
-            if (!mat.name?.includes("(lightmap)")) mat.name = material.name + " (lightmap)";
-            material = mat;
-            material.onBeforeCompile = this.onBeforeCompile;
-            this.clonedMaterials[index] = material;
-
-        }
-        return material;
-    }
-
-    private assignLightmapTexture(material: MaterialWithLightmap) {
-        if (!material) return;
-        if (material instanceof MeshPhysicalMaterial && material.transmission > 0) {
-            return;
-        }
-        const hasChanged = material.lightMap !== this.lightmapTexture || material[$lightmapVersion] !== material.version;
-        if (!hasChanged) {
-            return;
-        }
-
-        if (debug) console.log(`Assigning lightmap texture ${this.renderer.name}: '${material.name}' (${material.version} ${material[$lightmapVersion]})`);
-
-        // assign the lightmap
-        material.lightMap = this.lightmapTexture;
-        material.needsUpdate = true;
-        // store the version of the material
-        material[$lightmapVersion] = material.version;
-    }
-
-    private onBeforeCompile = (shader: WebGLProgramParametersWithUniforms, _) => {
-        if (debug === "verbose") console.log("Lightmaps, before compile\n", shader)
-        this.lightmapScaleOffsetUniform.value = this.lightmapScaleOffset;
-        this.lightmapUniform.value = this.lightmapTexture;
-        shader.uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
-    }
-
     private setLightmapDebugMaterial() {
+        const so = this.lightmapScaleOffset;
 
         // debug lightmaps
         this.gameObject["material"] = new ShaderMaterial({
@@ -190,7 +180,6 @@ export class RendererLightmap {
             fragmentShader: `
                 uniform sampler2D lightMap;
                 uniform float lightMapIntensity;
-                uniform vec4 lightmapScaleOffset;
                 varying vec2 vUv1;
 
                 // took from threejs 05fc79cd52b79e8c3e8dec1e7dca72c5c39983a4
@@ -199,8 +188,8 @@ export class RendererLightmap {
                 }
 
                 void main() {
-                    vec2 lUv = vUv1.xy * lightmapScaleOffset.xy + vec2(lightmapScaleOffset.z, (1. - (lightmapScaleOffset.y + lightmapScaleOffset.w)));
-                    
+                    vec2 lUv = vUv1.xy * vec2(${so.x.toFixed(6)}, ${so.y.toFixed(6)}) + vec2(${so.z.toFixed(6)}, ${(1 - so.y - so.w).toFixed(6)});
+
                     vec4 lightMapTexel = texture2D( lightMap, lUv);
                     gl_FragColor = lightMapTexel;
                     gl_FragColor.a = 1.;
@@ -209,4 +198,4 @@ export class RendererLightmap {
             defines: { USE_LIGHTMAP: '' }
         });
     }
-}
+}

package/src/engine-components/SeeThrough.ts

@@ -6,12 +6,12 @@ import { serializable } from "../engine/engine_serialization_decorator.js";
 import { getTempVector } from "../engine/engine_three_utils.js";
 import { getParam } from "../engine/engine_utils.js";
 import { USDObject, USDZExporterContext } from "./api.js";
+import type { Camera } from "./Camera.js";
 import { Behaviour } from "./Component.js";
 import { IUSDExporterExtension } from "./export/usdz/Extension.js";
 import { USDZExporter } from "./export/usdz/USDZExporter.js";
-import { Renderer } from "./Renderer.js";
-import type { Camera } from "./Camera.js";
 import type { OrbitControls } from "./OrbitControls.js";
+import { Renderer } from "./Renderer.js";
 
 const debugSeeThrough = getParam("debugseethrough");
 

package/src/engine-components/VideoPlayer.ts

@@ -604,11 +604,17 @@ export class VideoPlayer extends Behaviour {
             const videoMaterial = this._videoMaterial as any;
 
             if (!this.targetMaterialProperty) {
+                if (debug && videoMaterial[fieldName] === undefined) {
+                    console.warn(`The target material does not have a '${fieldName}' property, video might not render correctly.`);
+                }
                 videoMaterial[fieldName] = this._videoTexture;
             }
             else {
                 switch (this.targetMaterialProperty) {
                     default:
+                        if (debug && videoMaterial[this.targetMaterialProperty] === undefined) {
+                            console.warn(`The target material does not have a '${this.targetMaterialProperty}' property, video might not render correctly.`);
+                        }
                         videoMaterial[fieldName] = this._videoTexture;
                         break;
                     // doesnt render:

package/src/engine-components/utils/OpenURL.ts

@@ -19,6 +19,7 @@ export enum OpenURLMode {
  * OpenURL behaviour opens a URL in a new tab or window when the object (or any if it's children) is clicked.
  * 
  * @category Interactivity
+ * @category Web
  * @group Components
  */
 export class OpenURL extends Behaviour implements IPointerClickHandler {

package/src/engine-components/web/CursorFollow.ts

@@ -87,6 +87,7 @@ const debug = getParam("debugcursor");
  *
  * @summary Makes objects follow the cursor/touch position in 3D space
  * @category Interactivity
+ * @category Web
  * @group Components
  * @component
  */

package/src/engine-components/web/ScrollFollow.ts

@@ -66,6 +66,7 @@ type ScrollFollowEvent = {
  * 
  * @summary Links scroll position to target objects
  * @category Web
+ * @category Interaction
  * @group Components
  * @component
  */

package/src/engine-components/webxr/WebARCameraBackground.ts

@@ -22,13 +22,22 @@ import { Behaviour } from "../Component.js";
 const debug = getParam("debugarcamera");
 
 /**
- * WebARCameraBackground is a component that allows to display the camera feed as a background in an AR session to more easily blend the real world with the virtual world or applying effects to the camera feed.
- * 
+ * WebARCameraBackground is a component that allows to display the camera feed as a background in an AR session to more easily blend the real world with the virtual world or applying effects to the camera feed.  
+ *
+ * This component automatically requests `camera-access` permission when entering AR mode, which is required to:  
+ * - Display the real-world camera feed as a background  
+ * - Include the camera feed in AR screenshots taken with {@link screenshot2}  
+ *
+ * **Note**: If you want to take AR screenshots with the camera feed but don't need to display it as a background,   
+ * you can still add this component to your scene (it will request camera access) or manually request the  
+ * `camera-access` feature in your `onBeforeXR` method.  
+ *
  * - Example: https://samples.needle.tools/ar-camera-background
- * 
+ *
  * @summary Displays the camera feed as background in WebAR sessions
  * @category XR
  * @group Components
+ * @see {@link screenshot2} for taking screenshots in AR (requires camera access for camera feed compositing)
  */
 export class WebARCameraBackground extends Behaviour {
 

package/src/engine-components/webxr/WebXR.ts

@@ -59,6 +59,7 @@ const debugQuicklook = getParam("debugusdz");
  * @see {@link XRControllerMovement} for VR locomotion
  * @see {@link WebARSessionRoot} for AR session configuration
  * @see {@link Avatar} for networked user avatars
+ * @see {@link screenshot2} for taking screenshots in XR (including AR camera feed compositing)
  * @link https://engine.needle.tools/docs/xr.html
  * @link https://engine.needle.tools/samples/?overlay=samples&tag=xr
  * @link https://engine.needle.tools/samples/collaborative-sandbox