@needle-tools/engine 4.13.1 → 4.14.0-next.31f837e

package/src/engine/engine_utils_screenshot.ts

@@ -32,6 +32,7 @@ declare type ScreenshotImageMimeType = "image/webp" | "image/png" | "image/jpeg"
  * const dataUrl = screenshot();
  * saveImage(dataUrl, "screenshot.png");
  * ```
+ * 
  */
 export function screenshot(context?: Context, width?: number, height?: number, mimeType: ScreenshotImageMimeType = "image/webp", camera?: Camera | null): string | null {
     return screenshot2({ context, width, height, mimeType, camera });
@@ -106,12 +107,30 @@ export declare type ScreenshotOptionsBlob = ScreenshotOptions & {
 }
 
 export declare type ScreenshotOptionsShare = ScreenshotOptions & {
+    /**
+     * Set `{ type: "share" }` to share the screenshot using the Web Share API. The promise will resolve with the blob of the screenshot and whether it was shared successfully or not. Note that the Web Share API is only available in secure contexts (HTTPS) and on some platforms.
+     */
     type: "share",
+    /**
+     * The filename to use when sharing the screenshot. If not provided, a default filename will be used.
+     */
     filename?: string,
+    /**
+     * The mime type of the shared file. If not provided, the mime type will be inferred from the screenshot options or default to "image/png".
+     */
     file_type?: ScreenshotImageMimeType,
 
+    /**
+     * The title to use when sharing the screenshot. This is optional and may not be supported by all platforms.
+     */
     title?: string,
+    /**
+     * The text to use when sharing the screenshot. This is optional and may not be supported by all platforms.
+     */
     text?: string,
+    /**
+     * The URL to use when sharing the screenshot. This is optional and may not be supported by all platforms.
+     */
     url?: string,
 }
 
@@ -120,36 +139,241 @@ declare type ScreenshotOptionsShareReturnType = {
     shared: boolean,
 }
 
-/** 
- * Take a screenshot from the current scene and return a {@link Texture}. This can applied to a surface in 3D space.
- * @param opts Provide `{ type: "texture" }` to get a texture instead of a data url.
+/**
+ * Take a screenshot from the current scene and return a {@link Texture}. This can be applied to a surface in 3D space.  
+ * @param opts Provide `{ type: "texture" }` to get a texture instead of a data url.  
  * @returns The texture of the screenshot. Returns null if the screenshot could not be taken.
+ * @category Utilities
+ * @example
+ * ```ts
+ * // Create a texture from the current view
+ * const screenshotTexture = screenshot2({ type: "texture", width: 512, height: 512 });
+ * if (screenshotTexture) {
+ *   myMaterial.map = screenshotTexture;
+ *   myMaterial.needsUpdate = true;
+ * }
+ *
+ * // Update an existing texture
+ * const existingTexture = new Texture();
+ * screenshot2({ type: "texture", target: existingTexture, transparent: true });
+ * ```
  */
 export function screenshot2(opts: ScreenshotOptionsTexture): Texture | null;
+
 /**
- * Take a screenshot from the current scene.  
- * @param opts
- * @returns The data url of the screenshot. Returns null if the screenshot could not be taken.
- * ```ts	
- * const res = screenshot2({
- *    width: 1024,
+ * Take a screenshot from the current scene and return a data URL string.
+ *
+ * @param opts Screenshot options. All properties are optional.
+ * @returns The data URL of the screenshot (e.g., "data:image/png;base64,..."). Returns null if the screenshot could not be taken.
+ * @category Utilities
+ *
+ * @example Basic screenshot
+ * ```ts
+ * // Take a simple screenshot with default settings
+ * const dataUrl = screenshot2({});
+ * console.log(dataUrl); // "data:image/webp;base64,..."
+ * ```
+ *
+ * @example High-resolution screenshot with transparent background
+ * ```ts
+ * const dataUrl = screenshot2({
+ *   width: 2048,
+ *   height: 2048,
+ *   mimeType: "image/png",
+ *   transparent: true,
+ *   trim: true, // Remove transparent edges
+ * });
+ * ```
+ *
+ * @example Screenshot with custom background color
+ * ```ts
+ * import { Color } from "three";
+ *
+ * const dataUrl = screenshot2({
+ *   width: 1024,
+ *   height: 1024,
+ *   background: new Color(0x00ff00), // Green background
+ * });
+ * ```
+ *
+ * @example Download screenshot automatically
+ * ```ts
+ * screenshot2({
+ *   width: 1920,
+ *   height: 1080,
+ *   mimeType: "image/jpeg",
+ *   download_filename: "my-scene.jpg",
+ * });
+ * ```
+ *
+ * @example Manual download using saveImage
+ * ```ts
+ * const dataUrl = screenshot2({
+ *   width: 1024,
+ *   height: 1024,
+ *   mimeType: "image/webp",
+ *   transparent: true,
+ * });
+ * if (dataUrl) {
+ *   saveImage(dataUrl, "screenshot.webp");
+ * }
+ * ```
+ *
+ * @example Screenshot from specific camera
+ * ```ts
+ * const myCamera = this.gameObject.getComponent(Camera);
+ * const dataUrl = screenshot2({
+ *   camera: myCamera,
+ *   width: 1024,
  *   height: 1024,
- *  mimeType: "image/webp",
- * transparent: true,
- * })
- * // use saveImage to download the image
- * saveImage(res, "screenshot.webp");
+ * });
  * ```
  */
 export function screenshot2(opts: ScreenshotOptionsDataUrl): string | null;
 
 /**
- * Take a screenshot asynchronously from the current scene.
- * @returns A promise that resolves with the blob of the screenshot. Returns null if the screenshot could not be taken.
- * @param {ScreenshotOptionsBlob} opts Set `{ type: "blob" }` to get a blob instead of a data url.
+ * Take a screenshot asynchronously and return a Blob. This is useful when you need to process or upload the image data.  
+ *
+ * @param opts Set `{ type: "blob" }` to get a blob instead of a data url. All other {@link ScreenshotOptions} are also available.  
+ * @returns A Promise that resolves with the Blob of the screenshot. Returns null if the screenshot could not be taken.  
+ * @category Utilities
+ *
+ * @example Upload screenshot to server
+ * ```ts
+ * const blob = await screenshot2({ type: "blob", mimeType: "image/png" });
+ * if (blob) {
+ *   const formData = new FormData();
+ *   formData.append("screenshot", blob, "screenshot.png");
+ *   await fetch("/api/upload", { method: "POST", body: formData });
+ * }
+ * ```
+ *
+ * @example Save blob to file (browser download)
+ * ```ts
+ * const blob = await screenshot2({
+ *   type: "blob",
+ *   width: 1920,
+ *   height: 1080,
+ *   transparent: true
+ * });
+ * if (blob) {
+ *   const url = URL.createObjectURL(blob);
+ *   saveImage(url, "screenshot.png");
+ *   URL.revokeObjectURL(url); // Clean up
+ * }
+ * ```
  */
 export function screenshot2(opts: ScreenshotOptionsBlob): Promise<Blob | null>;
+
+/**
+ * Take a screenshot and share it using the Web Share API (mobile-friendly).  
+ *
+ * **Note**: The Web Share API is only available in secure contexts (HTTPS) and may not be supported on all platforms/browsers.  
+ *
+ * @param opts Set `{ type: "share" }` to share the screenshot. Additional options like `filename`, `title`, `text`, and `url` can be provided.
+ * @returns A Promise that resolves with an object containing the blob and whether it was successfully shared.
+ * @category Utilities
+ *
+ * @example Share screenshot on mobile
+ * ```ts
+ * const result = await screenshot2({
+ *   type: "share",
+ *   filename: "my-creation.png",
+ *   title: "Check out my 3D scene!",
+ *   text: "I created this with Needle Engine",
+ *   url: "https://engine.needle.tools",
+ *   mimeType: "image/png",
+ * });
+ *
+ * if (result.shared) {
+ *   console.log("Screenshot shared successfully!");
+ * } else {
+ *   console.log("User cancelled or sharing not supported");
+ * }
+ * ```
+ *
+ * @example Share with fallback
+ * ```ts
+ * const result = await screenshot2({
+ *   type: "share",
+ *   filename: "screenshot.webp",
+ *   file_type: "image/webp",
+ * });
+ *
+ * if (!result.shared && result.blob) {
+ *   // Fallback: download the image instead
+ *   const url = URL.createObjectURL(result.blob);
+ *   saveImage(url, "screenshot.webp");
+ *   URL.revokeObjectURL(url);
+ * }
+ * ```
+ */
 export function screenshot2(opts: ScreenshotOptionsShare): Promise<ScreenshotOptionsShareReturnType>;
+
+/**
+ * Take a screenshot from the current scene with advanced options.  
+ *
+ * This is the main screenshot function in Needle Engine with support for multiple output formats:  
+ * - **Data URL** (default): Returns a base64-encoded string suitable for img src attributes
+ * - **Texture**: Returns a Three.js Texture that can be applied to materials
+ * - **Blob**: Returns a Blob for uploading or processing (async)
+ * - **Share**: Uses the Web Share API to share the screenshot (async, mobile-friendly)
+ *
+ * @param opts Screenshot options. Use the `type` property to specify output format. See {@link ScreenshotOptions} for all available options.
+ * @returns Depending on the `type` option:
+ * - Data URL (string) when `type` is undefined or not specified
+ * - Texture when `type: "texture"`
+ * - Promise<Blob | null> when `type: "blob"`
+ * - Promise<{blob, shared}> when `type: "share"`
+ *
+ * Returns null (or Promise resolving to null) if the screenshot could not be taken.
+ *
+ * @category Utilities
+ *
+ * @example WebXR / AR Screenshots
+ * ```ts
+ * // Screenshots work automatically in WebXR sessions
+ * // The camera feed will be composited with your 3D content
+ * const dataUrl = screenshot2({
+ *   width: 1920,
+ *   height: 1080
+ * });
+ * ```
+ *
+ * **Note for AR Screenshots**: To include the device camera feed in AR screenshots, you need to request camera access.
+ * This is done automatically when you use {@link WebARCameraBackground} component in your scene.
+ * If you're not using WebARCameraBackground, you can request camera access manually:
+ * ```ts
+ * export class MyComponent extends Behaviour {
+ *   onBeforeXR(mode: XRSessionMode, args: XRSessionInit): void {
+ *     if (mode === "immersive-ar") {
+ *       args.optionalFeatures = args.optionalFeatures || [];
+ *       args.optionalFeatures.push('camera-access');
+ *     }
+ *   }
+ * }
+ * ```
+ * Without camera access, AR screenshots will only show your 3D content without the real-world background.
+ *
+ * @example Combining multiple options
+ * ```ts
+ * // High-res transparent screenshot with custom camera
+ * const myCamera = this.gameObject.getComponent(Camera);
+ * const texture = screenshot2({
+ *   type: "texture",
+ *   camera: myCamera,
+ *   width: 2048,
+ *   height: 2048,
+ *   transparent: true,
+ *   trim: true,
+ *   render_events: true, // Ensure reflection probes are updated
+ * });
+ * ```
+ *
+ * @see {@link screenshot} for a simpler alternative with fewer options
+ * @see {@link saveImage} for downloading data URLs
+ * @see {@link ScreenshotOptions} for all available options
+ */
 export function screenshot2(opts: ScreenshotOptionsDataUrl | ScreenshotOptionsTexture | ScreenshotOptionsBlob | ScreenshotOptionsShare)
     : Texture | string | null | Promise<Blob | null> | Promise<ScreenshotOptionsShareReturnType> {
 

package/src/engine/engine_utils_screenshot.xr.ts

@@ -5,7 +5,7 @@ import { isDevEnvironment, showBalloonError } from "./debug/index.js";
 // Adapted from WebARCameraBackground
 
 /**
- * Assigns the camera feed to a texture - this must be called during the render loop
+ * Assigns the camera feed to a texture - this must be called during the render loop.  
  */
 export function updateTextureFromXRFrame(renderer: WebGLRenderer, target: Texture): boolean {
 

package/src/engine/extensions/NEEDLE_techniques_webgl.ts

@@ -621,6 +621,9 @@ function createUniformProperties(material: CustomShader) {
                 case "_Color":
                     defineProperty("color", key);
                     break;
+                case "_map":
+                    defineProperty("map", key);
+                    break;
                 // case "_Metallic":
                 //     defineProperty("metalness", key);
                 //     break;

package/src/engine/xr/NeedleXRSession.ts

@@ -279,7 +279,12 @@ const $initialFov = Symbol("initial-fov");
  * The XRRig can be accessed via the `rig` property  
  * Set a custom XRRig via `NeedleXRSession.addRig(...)` or `NeedleXRSession.removeRig(...)`  
  * By default the active XRRig with the highest priority in the scene is used
+ *
+ * ### Screenshots in XR
+ * Screenshots work automatically during XR sessions, including AR camera feed compositing. See {@link screenshot2} for more information.
+ *
  * @category XR
+ * @see {@link screenshot2} for taking screenshots in XR sessions
  */
 export class NeedleXRSession implements INeedleXRSession {
 

package/src/engine-components/ContactShadows.ts

@@ -13,9 +13,9 @@ import { HideFlags, IGameObject, Vec3 } from "../engine/engine_types.js";
 import { getParam } from "../engine/engine_utils.js"
 import { setCustomVisibility } from "../engine/js-extensions/Layers.js";
 import { Behaviour, GameObject } from "./Component.js";
-import type { ShadowCatcher } from "./ShadowCatcher.js";
 import type { Light } from "./Light.js";
 import type { Renderer } from "./Renderer.js";
+import type { ShadowCatcher } from "./ShadowCatcher.js";
 
 const debug = getParam("debugcontactshadows");
 

package/src/engine-components/ReflectionProbe.ts

@@ -1,4 +1,4 @@
-import { CubeReflectionMapping, CubeTexture, EquirectangularReflectionMapping, LinearSRGBColorSpace, Material, MeshBasicMaterial, Object3D, SRGBColorSpace, Texture, Vector3 } from "three";
+import { Color, CubeReflectionMapping, CubeTexture, EquirectangularReflectionMapping, LinearSRGBColorSpace, Material, MeshBasicMaterial, MeshStandardMaterial, Object3D, SRGBColorSpace, Texture, Vector3 } from "three";
 
 import { isDevEnvironment, showBalloonWarning } from "../engine/debug/index.js";
 import { serializable } from "../engine/engine_serialization.js";
@@ -7,13 +7,12 @@ import type { IRenderer } from "../engine/engine_types.js";
 import { getParam } from "../engine/engine_utils.js";
 import { BoxHelperComponent } from "./BoxHelperComponent.js";
 import { Behaviour } from "./Component.js";
+import { MaterialPropertyBlock } from "../engine/engine_materialpropertyblock.js";
 
 export const debug = getParam("debugreflectionprobe");
 const disable = getParam("noreflectionprobe");
 
 const $reflectionProbeKey = Symbol("reflectionProbeKey");
-const $originalMaterial = Symbol("original material");
-
 /**
  * The [ReflectionProbe](https://engine.needle.tools/docs/api/ReflectionProbe) provides environment reflection data to materials within its defined area.
  * Use for chrome-like materials that need accurate environment reflections.  
@@ -38,12 +37,13 @@ const $originalMaterial = Symbol("original material");
  * @see {@link Renderer} for material and rendering control
  * @see {@link Light} for scene lighting
  */
+
 export class ReflectionProbe extends Behaviour {
 
     private static _probes: Map<Context, ReflectionProbe[]> = new Map();
 
     static isUsingReflectionProbe(material: Material) {
-        return !!(material[$reflectionProbeKey] || material[$originalMaterial]?.[$reflectionProbeKey]);
+        return !!(material as any)[$reflectionProbeKey];
     }
 
     public static get(object: Object3D | null | undefined, context: Context, isAnchor: boolean, anchor?: Object3D): ReflectionProbe | null {
@@ -55,7 +55,6 @@ export class ReflectionProbe extends Behaviour {
                 if (!probe.__didAwake) probe.__internalAwake();
                 if (probe.activeAndEnabled) {
                     if (anchor) {
-                        // test if anchor is reflection probe object
                         if (probe.gameObject === anchor) {
                             return probe;
                         }
@@ -72,13 +71,9 @@ export class ReflectionProbe extends Behaviour {
         return null;
     }
 
-
-
     private _texture!: Texture;
 
-    // @serializable(Texture)
     set texture(tex: Texture) {
-
         if (this._texture === tex) return;
         this._texture = tex;
 
@@ -86,7 +81,6 @@ export class ReflectionProbe extends Behaviour {
 
         if (tex) {
             if (tex instanceof CubeTexture) {
-                // cube textures use CubeReflectionMapping by default
             }
             else if (tex.mapping !== EquirectangularReflectionMapping) {
                 tex.mapping = EquirectangularReflectionMapping;
@@ -146,29 +140,14 @@ export class ReflectionProbe extends Behaviour {
         }
     }
 
-
-    // when objects are rendered and they share material
-    // and some need reflection probe and some don't
-    // we need to make sure we don't override the material but use a copy
-
-    private static _rendererMaterialsCache: Map<IRenderer, Array<{ material: Material, copy: Material, originalVersion: number }>> = new Map();
-
     onSet(_rend: IRenderer) {
         if (disable) return;
         if (!this.enabled) return;
         if (_rend.sharedMaterials?.length <= 0) return;
         if (!this.texture) return;
 
-        let rendererCache = ReflectionProbe._rendererMaterialsCache.get(_rend);
-        if (!rendererCache) {
-            rendererCache = [];
-            ReflectionProbe._rendererMaterialsCache.set(_rend, rendererCache);
-        }
-
-        // TODO: dont clone material for every renderer that uses reflection probes, we can do it once per material when they use the same reflection texture
+        const object = _rend.gameObject as unknown as Object3D;
 
-        // need to make sure materials are not shared when using reflection probes
-        // otherwise some renderers outside of the probe will be affected or vice versa
         for (let i = 0; i < _rend.sharedMaterials.length; i++) {
             const material = _rend.sharedMaterials[i];
             if (!material) {
@@ -178,75 +157,24 @@ export class ReflectionProbe extends Behaviour {
                 continue;
             }
 
-            if (material instanceof MeshBasicMaterial) {
-                continue;
-            }
-
-            let cached = rendererCache[i];
-
-            // make sure we have the currently assigned material cached (and an up to date clone of that)
-            // Compare against the stored original version, not the clone's version (which gets modified by needsUpdate)
-            // This prevents cloning materials every frame when onUnset restores the original material
-            const isCachedInstance = material === cached?.copy;
-            const hasChanged = !cached || cached.material.uuid !== material.uuid || cached.originalVersion !== material.version;
-            if (!isCachedInstance && hasChanged) {
-                if (debug) {
-                    let reason = "";
-                    if (!cached) reason = "not cached";
-                    else if (cached.material !== material) reason = "reference changed; cached instance?: " + isCachedInstance;
-                    else if (cached.originalVersion !== material.version) reason = "version changed";
-                    console.warn("Cloning material", material.name, material.version, "Reason:", reason, "\n", material.uuid, "\n", cached?.copy.uuid, "\n", _rend.name);
-                }
-
-                const clone = material.clone();
-                clone.version = material.version;
-
-                if (cached) {
-                    cached.copy = clone;
-                    cached.material = material;
-                    cached.originalVersion = material.version;
-                }
-                else {
-                    cached = {
-                        material: material,
-                        copy: clone,
-                        originalVersion: material.version
-                    };
-                    rendererCache.push(cached);
-                }
-
-                clone[$reflectionProbeKey] = this;
-                clone[$originalMaterial] = material;
-
-                if (debug) console.log("Set reflection", _rend.name, _rend.guid);
-            }
-
-            // See NE-4771 and NE-4856
-            if (cached && cached.copy) {
-                cached.copy.onBeforeCompile = material.onBeforeCompile;
-            }
-
-            /** this is the material that we copied and that has the reflection probe */
-            const copy = cached?.copy;
+            if (debug) console.log("Setting reflection probe on material", material.name, "for renderer", _rend.name);
 
-            if ("envMap" in copy) {
-                if (copy.envMap !== this.texture) {  // Only update if changed
-                    copy.envMap = this.texture;
-                    copy.needsUpdate = true;
-                }       
-            }
+            const propertyBlock = MaterialPropertyBlock.get(object);
+            propertyBlock.setOverride("envMap", this.texture);
+            // propertyBlock.setOverride("color", new Color(0xff0000));
 
-            _rend.sharedMaterials[i] = copy;
+            (material as any)[$reflectionProbeKey] = true;
         }
     }
 
     onUnset(_rend: IRenderer) {
-        const rendererCache = ReflectionProbe._rendererMaterialsCache.get(_rend);
-        if (rendererCache) {
-            for (let i = 0; i < rendererCache.length; i++) {
-                const cached = rendererCache[i];
-                _rend.sharedMaterials[i] = cached.material;
+        // const object = _rend.gameObject as unknown as Object3D;
+
+        for (let i = 0; i < _rend.sharedMaterials.length; i++) {
+            const material = _rend.sharedMaterials[i];
+            if (material) {
+                delete (material as any)[$reflectionProbeKey];
             }
         }
     }
-}
+}