@babylonjs/core 7.43.0 → 7.44.0

package/XR/features/WebXRDepthSensing.d.ts

@@ -3,9 +3,9 @@ import type { WebXRSessionManager } from "../webXRSessionManager";
 import { WebXRAbstractFeature } from "./WebXRAbstractFeature";
 import { Observable } from "../../Misc/observable";
 import type { Nullable } from "../../types";
-import { InternalTexture } from "../../Materials/Textures/internalTexture";
+import type { InternalTexture } from "../../Materials/Textures/internalTexture";
 export type WebXRDepthUsage = "cpu" | "gpu";
-export type WebXRDepthDataFormat = "ushort" | "float";
+export type WebXRDepthDataFormat = "ushort" | "float" | "luminance-alpha";
 /**
  * Options for Depth Sensing feature
  */
@@ -18,6 +18,23 @@ export interface IWebXRDepthSensingOptions {
      * The desired depth sensing data format for the session
      */
     dataFormatPreference: WebXRDepthDataFormat[];
+    /**
+     * Depth sensing will be enabled on all materials per default, if the GPU variant is enabled.
+     * If you just want to use the texture or the CPU variant instead set this to true.
+     */
+    disableDepthSensingOnMaterials?: boolean;
+    /**
+     * If set to true the occluded pixels will not be discarded but the pixel color will be changed based on the occlusion factor
+     * Enabling this will lead to worse performance but slightly better outcome.
+     * It is possible we will change this in the future to look even better.
+     */
+    useToleranceFactorForDepthSensing?: boolean;
+    /**
+     * If set to true the texture will be set to be used for visualization.
+     * In this case it will probably NOT work correctly on the materials.
+     * So be aware that, for the time being, you can only use one or the other.
+     */
+    prepareTextureForVisualization?: boolean;
 }
 type GetDepthInMetersType = (x: number, y: number) => number;
 /**
@@ -29,10 +46,12 @@ export declare class WebXRDepthSensing extends WebXRAbstractFeature {
     private _width;
     private _height;
     private _rawValueToMeters;
+    private _textureType;
     private _normDepthBufferFromNormView;
     private _cachedDepthBuffer;
     private _cachedWebGLTexture;
     private _cachedDepthImageTexture;
+    private _onCameraObserver;
     /**
      * Width of depth data. If depth data is not exist, returns null.
      */
@@ -60,6 +79,7 @@ export declare class WebXRDepthSensing extends WebXRAbstractFeature {
     /**
      * Latest cached InternalTexture which containing depth buffer information.
      * This can be used when the depth usage is "gpu".
+     * @deprecated This will be removed in the future. Use latestDepthImageTexture
      */
     get latestInternalTexture(): Nullable<InternalTexture>;
     /**
@@ -102,6 +122,7 @@ export declare class WebXRDepthSensing extends WebXRAbstractFeature {
      * @returns true if successful.
      */
     attach(force?: boolean | undefined): boolean;
+    detach(): boolean;
     /**
      * Dispose this feature and all of the resources attached
      */
@@ -114,5 +135,6 @@ export declare class WebXRDepthSensing extends WebXRAbstractFeature {
      * @returns augmentation object for the xr session init object.
      */
     getXRSessionInitExtension(): Promise<Partial<XRSessionInit>>;
+    private _getInternalTextureFromDepthInfo;
 }
 export {};

package/XR/features/WebXRDepthSensing.js

@@ -6,7 +6,222 @@ import { Texture } from "../../Materials/Textures/texture.js";
 import { Observable } from "../../Misc/observable.js";
 
 import { WebGLHardwareTexture } from "../../Engines/WebGL/webGLHardwareTexture.js";
-import { InternalTexture } from "../../Materials/Textures/internalTexture.js";
+import { MaterialPluginBase } from "../../Materials/materialPluginBase.js";
+import { MaterialDefines } from "../../Materials/materialDefines.js";
+import { PBRBaseMaterial } from "../../Materials/PBR/pbrBaseMaterial.js";
+import { RegisterMaterialPlugin } from "../../Materials/materialPluginManager.js";
+import { Matrix } from "../../Maths/math.vector.js";
+class DepthSensingMaterialDefines extends MaterialDefines {
+    constructor() {
+        super(...arguments);
+        /**
+         * Is the feature enabled
+         */
+        this.DEPTH_SENSING = false;
+        /**
+         * Is the texture type provided as a texture array
+         */
+        this.DEPTH_SENSING_TEXTURE_ARRAY = false;
+        /**
+         * Is the texture type provided as Alpha-Luminance (unpacked differently on the shader)
+         */
+        this.DEPTH_SENSING_TEXTURE_AL = false;
+        /**
+         * Should the shader discard the pixel if the depth is less than the asset depth
+         * Will lead to better performance. the other variant is to change the color based on a tolerance factor
+         */
+        this.DEPTH_SENSING_DISCARD = true;
+    }
+}
+let isPluginEnabled = false;
+let depthTexture = null;
+let alphaLuminanceTexture = false;
+const screenSize = { width: 512, height: 512 };
+const shaderViewport = { x: 0, y: 0, width: 1, height: 1 };
+let globalRawValueToMeters = 1;
+let viewIndex = 0;
+let enableDiscard = true;
+const uvTransform = Matrix.Identity();
+const managedMaterialPlugins = [];
+/**
+ * @internal
+ */
+class WebXRDepthSensingMaterialPlugin extends MaterialPluginBase {
+    /** @internal */
+    _markAllDefinesAsDirty() {
+        this._enable(this._isEnabled);
+        this.markAllDefinesAsDirty();
+    }
+    /**
+     * Gets whether the mesh debug plugin is enabled in the material.
+     */
+    get isEnabled() {
+        return this._isEnabled;
+    }
+    /**
+     * Sets whether the mesh debug plugin is enabled in the material.
+     * @param value enabled
+     */
+    set isEnabled(value) {
+        if (this._isEnabled === value) {
+            return;
+        }
+        this._isEnabled = value;
+        this._markAllDefinesAsDirty();
+    }
+    /**
+     * Gets a boolean indicating that the plugin is compatible with a given shader language.
+     * @param shaderLanguage The shader language to use.
+     * @returns true if the plugin is compatible with the shader language
+     */
+    isCompatible(shaderLanguage) {
+        switch (shaderLanguage) {
+            case 0 /* ShaderLanguage.GLSL */:
+                return true;
+            default:
+                // no webgpu for webxr yet, however - if this is not true the plugin fails to load.
+                // webxr is currently only supported on webgl, and the plugin is disabled per default.
+                return true;
+        }
+    }
+    constructor(material) {
+        super(material, "DepthSensing", 222, new DepthSensingMaterialDefines());
+        this._isEnabled = false;
+        this._varColorName = material instanceof PBRBaseMaterial ? "finalColor" : "color";
+        managedMaterialPlugins.push(this);
+    }
+    /**
+     * Prepare the defines
+     * @param defines the defines
+     */
+    prepareDefines(defines) {
+        defines.DEPTH_SENSING = !!depthTexture && isPluginEnabled;
+        defines.DEPTH_SENSING_TEXTURE_ARRAY = depthTexture?.is2DArray ?? false;
+        defines.DEPTH_SENSING_TEXTURE_AL = alphaLuminanceTexture;
+        defines.DEPTH_SENSING_DISCARD = enableDiscard;
+    }
+    getUniforms() {
+        return {
+            // first, define the UBO with the correct type and size.
+            ubo: [
+                { name: "ds_invScreenSize", size: 2, type: "vec2" },
+                { name: "ds_rawValueToMeters", size: 1, type: "float" },
+                { name: "ds_viewIndex", size: 1, type: "float" },
+                { name: "ds_shaderViewport", size: 4, type: "vec4" },
+                { name: "ds_uvTransform", size: 16, type: "mat4" },
+            ],
+            // now, on the fragment shader, add the uniform itself in case uniform buffers are not supported by the engine
+            fragment: `#ifdef DEPTH_SENSING
+                uniform vec2 ds_invScreenSize;
+                uniform float ds_rawValueToMeters;
+                uniform float ds_viewIndex;
+                uniform vec4 ds_shaderViewport;
+                uniform mat4 ds_uvTransform;
+                #endif
+                `,
+        };
+    }
+    getSamplers(samplers) {
+        samplers.push("ds_depthSampler");
+    }
+    bindForSubMesh(uniformBuffer) {
+        if (isPluginEnabled && depthTexture) {
+            uniformBuffer.updateFloat2("ds_invScreenSize", 1 / screenSize.width, 1 / screenSize.height);
+            uniformBuffer.updateFloat("ds_rawValueToMeters", globalRawValueToMeters);
+            uniformBuffer.updateFloat("ds_viewIndex", viewIndex);
+            uniformBuffer.updateFloat4("ds_shaderViewport", shaderViewport.x, shaderViewport.y, shaderViewport.width, shaderViewport.height);
+            uniformBuffer.setTexture("ds_depthSampler", depthTexture);
+            uniformBuffer.updateMatrix("ds_uvTransform", uvTransform);
+        }
+    }
+    getClassName() {
+        return "DepthSensingMaterialPlugin";
+    }
+    getCustomCode(shaderType) {
+        return shaderType === "vertex"
+            ? {
+                CUSTOM_VERTEX_MAIN_BEGIN: `
+                #ifdef DEPTH_SENSING
+                #ifdef MULTIVIEW
+                    ds_viewIndexMultiview = float(gl_ViewID_OVR);
+                #endif
+                #endif
+                `,
+                CUSTOM_VERTEX_DEFINITIONS: `
+                #ifdef DEPTH_SENSING
+                #ifdef MULTIVIEW
+                    varying float ds_viewIndexMultiview;
+                #endif
+                #endif
+                `,
+            }
+            : {
+                CUSTOM_FRAGMENT_DEFINITIONS: `
+                    #ifdef DEPTH_SENSING
+                        #ifdef DEPTH_SENSING_TEXTURE_ARRAY
+                            uniform highp sampler2DArray ds_depthSampler;
+                        #else
+                            uniform sampler2D ds_depthSampler;
+                        #endif
+                        #ifdef MULTIVIEW
+                            varying float ds_viewIndexMultiview;
+                        #endif
+                    #endif
+                  `,
+                CUSTOM_FRAGMENT_MAIN_BEGIN: `
+#ifdef DEPTH_SENSING
+    #ifdef MULTIVIEW
+        float ds_viewIndexSet = ds_viewIndexMultiview;
+        vec2 ds_compensation = vec2(0.0, 0.0);
+    #else
+        float ds_viewIndexSet = ds_viewIndex;
+        vec2 ds_compensation = vec2(ds_viewIndexSet, 0.0);
+    #endif
+    vec2 ds_baseUv = gl_FragCoord.xy * ds_invScreenSize;
+    #ifdef DEPTH_SENSING_TEXTURE_ARRAY
+        vec2 ds_uv = ds_baseUv - ds_compensation;
+        vec3 ds_depthUv = vec3((ds_uvTransform * vec4(ds_uv, 0.0, 1.0)).xy, ds_viewIndexSet);
+    #else
+        vec2 ds_depthUv = (ds_uvTransform * vec4(ds_baseUv.x, 1.0 - ds_baseUv.y, 0.0, 1.0)).xy;
+    #endif
+    #ifdef DEPTH_SENSING_TEXTURE_AL
+        // from alpha-luminance - taken from the explainer
+        vec2 ds_alphaLuminance = texture(ds_depthSampler, ds_depthUv).ra;
+        float ds_cameraDepth = dot(ds_alphaLuminance, vec2(255.0, 256.0 * 255.0));
+    #else
+        float ds_cameraDepth = texture(ds_depthSampler, ds_depthUv).r;
+    #endif
+
+    ds_cameraDepth = ds_cameraDepth * ds_rawValueToMeters;
+
+    float ds_assetDepth = gl_FragCoord.z;
+    #ifdef DEPTH_SENSING_DISCARD
+    if(ds_cameraDepth < ds_assetDepth) {
+        discard;
+    }
+    #endif
+#endif  
+                  `,
+                CUSTOM_FRAGMENT_BEFORE_FRAGCOLOR: `
+#ifdef DEPTH_SENSING
+    #ifndef DEPTH_SENSING_DISCARD
+        const float ds_depthTolerancePerM = 0.005;
+        float ds_occlusion = clamp(1.0 - 0.5 * (ds_cameraDepth - ds_assetDepth) / (ds_depthTolerancePerM * ds_assetDepth) +
+            0.5, 0.0, 1.0);
+        ${this._varColorName} *= (1.0 - ds_occlusion);
+    #endif
+#endif                  
+                  `,
+            };
+    }
+    dispose(_forceDisposeTextures) {
+        const index = managedMaterialPlugins.indexOf(this);
+        if (index !== -1) {
+            managedMaterialPlugins.splice(index, 1);
+        }
+        super.dispose(_forceDisposeTextures);
+    }
+}
 /**
  * WebXR Feature for WebXR Depth Sensing Module
  * @since 5.49.1
@@ -56,31 +271,20 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
                 return "ushort";
             case "float32":
                 return "float";
+            case "unsigned-short":
+                return "ushort";
         }
     }
     /**
      * Latest cached InternalTexture which containing depth buffer information.
      * This can be used when the depth usage is "gpu".
+     * @deprecated This will be removed in the future. Use latestDepthImageTexture
      */
     get latestInternalTexture() {
         if (!this._cachedWebGLTexture) {
             return null;
         }
-        const engine = this._xrSessionManager.scene.getEngine();
-        const internalTexture = new InternalTexture(engine, 0 /* InternalTextureSource.Unknown */);
-        internalTexture.isCube = false;
-        internalTexture.invertY = false;
-        internalTexture._useSRGBBuffer = false;
-        internalTexture.format = this.depthDataFormat === "ushort" ? 2 : 5;
-        internalTexture.generateMipMaps = false;
-        internalTexture.type = this.depthDataFormat === "ushort" ? 5 : 1;
-        internalTexture.samplingMode = 7;
-        internalTexture.width = this.width ?? 0;
-        internalTexture.height = this.height ?? 0;
-        internalTexture._cachedWrapU = 1;
-        internalTexture._cachedWrapV = 1;
-        internalTexture._hardwareTexture = new WebGLHardwareTexture(this._cachedWebGLTexture, engine._gl);
-        return internalTexture;
+        return this._getInternalTextureFromDepthInfo();
     }
     /**
      * cached depth buffer
@@ -89,7 +293,7 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
         if (!this._cachedDepthBuffer) {
             return null;
         }
-        return this.depthDataFormat === "ushort" ? new Uint16Array(this._cachedDepthBuffer) : new Float32Array(this._cachedDepthBuffer);
+        return this.depthDataFormat === "float" ? new Float32Array(this._cachedDepthBuffer) : new Uint16Array(this._cachedDepthBuffer);
     }
     /**
      * Latest cached Texture of depth image which is made from the depth buffer data.
@@ -108,10 +312,12 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
         this._width = null;
         this._height = null;
         this._rawValueToMeters = null;
+        this._textureType = null;
         this._normDepthBufferFromNormView = null;
         this._cachedDepthBuffer = null;
         this._cachedWebGLTexture = null;
         this._cachedDepthImageTexture = null;
+        this._onCameraObserver = null;
         /**
          * Event that notify when `DepthInformation.getDepthInMeters` is available.
          * `getDepthInMeters` method needs active XRFrame (not available for cached XRFrame)
@@ -120,6 +326,7 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
         this.xrNativeFeatureName = "depth-sensing";
         // https://immersive-web.github.io/depth-sensing/
         Tools.Warn("depth-sensing is an experimental and unstable feature.");
+        enableDiscard = !options.useToleranceFactorForDepthSensing;
     }
     /**
      * attach this feature
@@ -136,14 +343,57 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
             return false;
         }
         this._glBinding = new XRWebGLBinding(this._xrSessionManager.session, this._xrSessionManager.scene.getEngine()._gl);
+        isPluginEnabled = !this.options.disableDepthSensingOnMaterials;
+        if (isPluginEnabled) {
+            managedMaterialPlugins.forEach((plugin) => {
+                plugin.isEnabled = true;
+            });
+            this._onCameraObserver = this._xrSessionManager.scene.onBeforeCameraRenderObservable.add((camera) => {
+                if (!isPluginEnabled) {
+                    return;
+                }
+                // make sure this is a webxr camera
+                if (camera.outputRenderTarget) {
+                    const viewport = camera.rigCameras.length > 0 ? camera.rigCameras[0].viewport : camera.viewport;
+                    screenSize.width = camera.outputRenderTarget.getRenderWidth() / (camera.rigParent ? camera.rigParent.rigCameras.length || 1 : 1);
+                    screenSize.height = camera.outputRenderTarget.getRenderHeight();
+                    shaderViewport.x = viewport.x;
+                    shaderViewport.y = viewport.y;
+                    shaderViewport.width = viewport.width;
+                    shaderViewport.height = viewport.height;
+                    // find the viewIndex
+                    if (camera.rigParent) {
+                        // should use the viewIndexes array!
+                        viewIndex = camera.isLeftCamera ? 0 : 1;
+                    }
+                }
+            });
+        }
         return true;
     }
+    detach() {
+        isPluginEnabled = false;
+        depthTexture = null;
+        this._cachedWebGLTexture = null;
+        this._cachedDepthBuffer = null;
+        managedMaterialPlugins.forEach((plugin) => {
+            plugin.isEnabled = false;
+        });
+        if (this._onCameraObserver) {
+            this._xrSessionManager.scene.onBeforeCameraRenderObservable.remove(this._onCameraObserver);
+        }
+        return super.detach();
+    }
     /**
      * Dispose this feature and all of the resources attached
      */
     dispose() {
         this._cachedDepthImageTexture?.dispose();
         this.onGetDepthInMetersAvailable.clear();
+        // cleanup
+        if (this._onCameraObserver) {
+            this._xrSessionManager.scene.onBeforeCameraRenderObservable.remove(this._onCameraObserver);
+        }
     }
     _onXRFrame(_xrFrame) {
         const referenceSPace = this._xrSessionManager.referenceSpace;
@@ -174,43 +424,63 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
         if (depthInfo === null) {
             return;
         }
-        const { data, width, height, rawValueToMeters, getDepthInMeters } = depthInfo;
+        const { data, width, height, rawValueToMeters, getDepthInMeters, normDepthBufferFromNormView } = depthInfo;
         this._width = width;
         this._height = height;
         this._rawValueToMeters = rawValueToMeters;
         this._cachedDepthBuffer = data;
+        globalRawValueToMeters = rawValueToMeters;
+        alphaLuminanceTexture = dataFormat === "luminance-alpha";
+        uvTransform.fromArray(normDepthBufferFromNormView.matrix);
         // to avoid Illegal Invocation error, bind `this`
         this.onGetDepthInMetersAvailable.notifyObservers(getDepthInMeters.bind(depthInfo));
         if (!this._cachedDepthImageTexture) {
-            this._cachedDepthImageTexture = RawTexture.CreateRTexture(null, width, height, this._xrSessionManager.scene, false, true, Texture.NEAREST_SAMPLINGMODE, 1);
+            this._cachedDepthImageTexture = RawTexture.CreateRTexture(null, width, height, this._xrSessionManager.scene, false, false, Texture.NEAREST_SAMPLINGMODE, 1);
+            depthTexture = this._cachedDepthImageTexture;
         }
+        let float32Array = null;
         switch (dataFormat) {
             case "ushort":
-                this._cachedDepthImageTexture.update(Float32Array.from(new Uint16Array(data)).map((value) => value * rawValueToMeters));
+            case "luminance-alpha":
+                float32Array = Float32Array.from(new Uint16Array(data));
                 break;
             case "float":
-                this._cachedDepthImageTexture.update(new Float32Array(data).map((value) => value * rawValueToMeters));
+                float32Array = new Float32Array(data);
                 break;
             default:
                 break;
         }
+        if (float32Array) {
+            if (this.options.prepareTextureForVisualization) {
+                float32Array = float32Array.map((val) => val * rawValueToMeters);
+            }
+            this._cachedDepthImageTexture.update(float32Array);
+        }
     }
     _updateDepthInformationAndTextureWebGLDepthUsage(webglBinding, view, dataFormat) {
         const depthInfo = webglBinding.getDepthInformation(view);
         if (depthInfo === null) {
             return;
         }
-        const { texture, width, height } = depthInfo;
+        const { texture, width, height, textureType, rawValueToMeters, normDepthBufferFromNormView } = depthInfo;
+        globalRawValueToMeters = rawValueToMeters;
+        alphaLuminanceTexture = dataFormat === "luminance-alpha";
+        uvTransform.fromArray(normDepthBufferFromNormView.matrix);
+        if (this._cachedWebGLTexture) {
+            return;
+        }
         this._width = width;
         this._height = height;
         this._cachedWebGLTexture = texture;
+        this._textureType = textureType;
         const scene = this._xrSessionManager.scene;
-        const engine = scene.getEngine();
-        const internalTexture = engine.wrapWebGLTexture(texture);
+        const internalTexture = this._getInternalTextureFromDepthInfo();
         if (!this._cachedDepthImageTexture) {
-            this._cachedDepthImageTexture = RawTexture.CreateRTexture(null, width, height, scene, false, true, Texture.NEAREST_SAMPLINGMODE, dataFormat === "ushort" ? 0 : 1);
+            this._cachedDepthImageTexture = RawTexture.CreateRTexture(null, width, height, scene, false, true, Texture.NEAREST_SAMPLINGMODE, dataFormat === "float" ? 1 : 0);
         }
         this._cachedDepthImageTexture._texture = internalTexture;
+        depthTexture = this._cachedDepthImageTexture;
+        this._xrSessionManager.scene.markAllMaterialsAsDirty(1);
     }
     /**
      * Extends the session init object if needed
@@ -231,10 +501,12 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
                 });
                 const dataFormats = this.options.dataFormatPreference.map((format) => {
                     switch (format) {
-                        case "ushort":
+                        case "luminance-alpha":
                             return "luminance-alpha";
                         case "float":
                             return "float32";
+                        case "ushort":
+                            return "unsigned-short";
                     }
                 });
                 resolve({
@@ -249,6 +521,27 @@ export class WebXRDepthSensing extends WebXRAbstractFeature {
             }
         });
     }
+    _getInternalTextureFromDepthInfo() {
+        const engine = this._xrSessionManager.scene.getEngine();
+        const dataFormat = this.depthDataFormat;
+        const textureType = this._textureType;
+        if (!this._width || !this._height || !this._cachedWebGLTexture) {
+            throw new Error("Depth information is not available");
+        }
+        const internalTexture = engine.wrapWebGLTexture(this._cachedWebGLTexture, false, 1, this._width || 256, this._height || 256);
+        internalTexture.isCube = false;
+        internalTexture.invertY = false;
+        internalTexture._useSRGBBuffer = false;
+        internalTexture.format = dataFormat === "luminance-alpha" ? 2 : 5;
+        internalTexture.generateMipMaps = false;
+        internalTexture.type =
+            dataFormat === "float" ? 1 : dataFormat === "ushort" ? 5 : 0;
+        internalTexture._cachedWrapU = 1;
+        internalTexture._cachedWrapV = 1;
+        internalTexture._hardwareTexture = new WebGLHardwareTexture(this._cachedWebGLTexture, engine._gl);
+        internalTexture.is2DArray = textureType === "texture-array";
+        return internalTexture;
+    }
 }
 /**
  * The module's name
@@ -263,4 +556,5 @@ WebXRDepthSensing.Version = 1;
 WebXRFeaturesManager.AddWebXRFeature(WebXRDepthSensing.Name, (xrSessionManager, options) => {
     return () => new WebXRDepthSensing(xrSessionManager, options);
 }, WebXRDepthSensing.Version, false);
+RegisterMaterialPlugin("WebXRDepthSensingMaterialPlugin", (material) => new WebXRDepthSensingMaterialPlugin(material));
 //# sourceMappingURL=WebXRDepthSensing.js.map