@needle-tools/engine 3.28.7-beta → 3.28.7-beta.1

package/src/engine-components/webxr/WebARCameraBackground.ts

@@ -1,225 +1,225 @@
-import { Behaviour } from "../Component.js";
-import { serializable } from "../../engine/engine_serialization_decorator.js";
-import { RGBAColor } from "../js-extensions/RGBAColor.js"
-import { WebXR } from "./WebXR.js";
-import { 
-    Scene, 
-    Texture, 
-    Mesh, MeshBasicMaterial, 
-    UniformsUtils, 
-    PlaneGeometry, 
-    ShaderLib, 
-    ShaderMaterial,
-    DoubleSide,
-    PerspectiveCamera,
-} from "three";
-
-export class WebARCameraBackground extends Behaviour {
-
-    awake(): void {
-        WebXR.OptionalFeatures_AR.push('camera-access');
-    }
-
-    @serializable()
-    public backgroundTint: RGBAColor = new RGBAColor(1,1,1,1);
-
-    public get background() {
-        return this.backgroundPlane;
-    }
-
-    private _preRender;
-
-    onEnable(): void {
-        this._preRender = this.preRender.bind(this);
-        this.context.pre_render_callbacks.push(this._preRender);
-
-        if (this.backgroundPlane) {
-            this.gameObject.add(this.backgroundPlane);
-            this.backgroundPlane.visible = false;
-        }
-    }
-
-    onDisable(): void {
-        this.context.pre_render_callbacks = this.context.pre_render_callbacks.filter(cb => cb !== this._preRender);
-
-        if (this.backgroundPlane)
-            this.gameObject.remove(this.backgroundPlane);
-    }
-
-    private backgroundPlane?: Mesh;
-    private threeTexture?: Texture;
-    private forceTextureInitialization = function() {
-        const material = new MeshBasicMaterial();
-        const geometry = new PlaneGeometry();
-        const scene = new Scene();
-        scene.add(new Mesh(geometry, material));
-        const camera = new PerspectiveCamera();
-    
-        return function forceTextureInitialization(renderer, texture) {
-          material.map = texture;
-          renderer.render(scene, camera);
-        };
-    }();
-
-    // TODO should only attach on session start, and detach on session end
-    private preRender() {
-        if (!this || !this.gameObject) return;
-
-        const xr = this.context.renderer.xr;
-        const frame = xr.getFrame();
-
-        if (frame) {
-            
-            // We're generating a new texture here, and force three to initialize it
-            // from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
-            if (!this.threeTexture && this.context.renderer) {
-                this.threeTexture = new Texture();
-                this.forceTextureInitialization(this.context.renderer, this.threeTexture);
-            }
-
-            // simple mesh and fullscreen shader to display the camera texture
-            // from three: WebGLBackground
-            if (this.backgroundPlane === undefined) {
-                this.backgroundPlane = makeFullscreenPlane(this.backgroundTint);
-                this.gameObject.add(this.backgroundPlane);
-            }
-
-            // WebXR Raw Camera Access - 
-            // we composite the camera texture into the scene background by rendering it first.
-            this.updateFromFrame(frame);
-        }
-
-        /*
-        if (this.planeMesh) {
-            this.planeMesh.visible = frame != null;
-        }
-        */
-    }
-
-    onBeforeRender(frame: XRFrame | null) {
-        this.updateFromFrame(frame);
-    }
-
-    updateFromFrame(frame: XRFrame | null) {
-        if (!frame) return;
-
-        // https://chromium.googlesource.com/chromium/src/+/7c5ac3c0f95b97cf12be95a5c1c0a8ff163246d8/third_party/webxr_test_pages/webxr-samples/proposals/camera-access-barebones.html
-        const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace()!);
-        if (pose) { 
-            for( const view of pose.views) {
-                // @ts-ignore
-                if ('camera' in view && view.camera) {
-                    const xrManager = this.context.renderer.xr;
-                    let binding = xrManager.getBinding();
-                    // not sure how / why this can be null, but we can recreate it here
-                    if (!binding) binding = new XRWebGLBinding( frame.session, this.context.renderer.getContext() );
-                    
-                    if (binding) {
-                        let glImage: WebGLTexture | null = null;
-                        if ('getCameraImage' in binding) {
-                            // @ts-ignore
-                            glImage = binding.getCameraImage(view.camera);
-
-                            // discussion on exactly this:
-                            // https://discourse.threejs.org/t/using-a-webgltexture-as-texture-for-three-js/46245/8
-                            // HACK from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
-                            const texProps = this.context.renderer.properties.get(this.threeTexture);
-                            texProps.__webglTexture = glImage;
-
-                            if (this.backgroundPlane) {
-                                //@ts-ignore
-                                this.backgroundPlane.setTexture(this.threeTexture);
-                                this.backgroundPlane.visible = true;
-                            }
-
-                            // TODO this would be a lot better but currently
-                            // setting color space doesn't work.
-                            // Plus we need to understand how we can supply a custom shader in
-                            // this case.
-                            /*
-                            if (this.threeTexture) {
-                                this.context.scene.background = this.threeTexture;
-                                this.threeTexture.colorSpace = NoColorSpace;
-                            }
-                            */
-                        }
-                    }
-                    else {
-                        // console.error(view.camera, xrManager)
-                    }
-                }
-                else {
-                    // console.error("NO CAMERA IN VIEW")
-                }
-            }
-        }
-        else {
-            // console.error(this.context.renderer.xr.getReferenceSpace(), frame);
-        }
-    }
-}
-// TODO tint could be an uniform
-const backgroundFragment: string = /* glsl */`
-uniform sampler2D t2D;
-
-varying vec2 vUv;
-
-void main() {
-
-    vec4 texColor = texture2D( t2D, vUv );
-    texColor.w = 1.0;
-
-    // inline sRGB decode
-    texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );
-
-    gl_FragColor = texColor * <backgroundTint>;
-
-    #include <tonemapping_fragment>
-    #include <encodings_fragment>
-
-}
-`;
-
-// not sure where we want to move this and in which form is best (extends Object3D?)
-export function makeFullscreenPlane(tint: RGBAColor ) {
-    const replacementTint = "vec4(" + tint.r.toFixed(3) + "," + tint.g.toFixed(3) + "," + tint.b.toFixed(3) + "," + tint.a.toFixed(3) + ")";
-    console.log(replacementTint);
-    const planeMesh = new Mesh(
-        new PlaneGeometry(2, 2),
-        // @ts-ignore
-        new ShaderMaterial({
-            name: 'BackgroundMaterial',
-            uniforms: UniformsUtils.clone( ShaderLib.background.uniforms ),
-            vertexShader: ShaderLib.background.vertexShader,
-            fragmentShader: backgroundFragment.replace("<backgroundTint>", replacementTint), // ShaderLib.background.fragmentShader,
-            side: DoubleSide,
-            depthTest: false,
-            depthWrite: false,
-            fog: false
-        })
-    );
-
-    planeMesh.geometry.deleteAttribute( 'normal' );
-
-    // add "map" material property so the renderer can evaluate it like for built-in materials
-    Object.defineProperty( planeMesh.material, 'map', {
-        get: function () {
-            return this.threeTexture;
-        }
-    } );
-
-    // Option 1: add the planeMesh to our scene for rendering.
-    // This is useful for applying custom shader effects on the background (instead of using the system composite)
-    planeMesh.renderOrder = -10000; // render first
-    planeMesh.layers.disableAll();
-    planeMesh.layers.enable(2); // ignore raycasts
-    planeMesh.frustumCulled = false;
-
-    // should be a class, for now lets just define a method for the weird way the texture needs to be set
-    // @ts-ignore
-    planeMesh.setTexture = function(texture) {
-        planeMesh.material.uniforms.t2D.value = texture;
-    }
-
-    return planeMesh;
+import { Behaviour } from "../Component.js";
+import { serializable } from "../../engine/engine_serialization_decorator.js";
+import { RGBAColor } from "../js-extensions/RGBAColor.js"
+import { WebXR } from "./WebXR.js";
+import { 
+    Scene, 
+    Texture, 
+    Mesh, MeshBasicMaterial, 
+    UniformsUtils, 
+    PlaneGeometry, 
+    ShaderLib, 
+    ShaderMaterial,
+    DoubleSide,
+    PerspectiveCamera,
+} from "three";
+
+export class WebARCameraBackground extends Behaviour {
+
+    awake(): void {
+        WebXR.OptionalFeatures_AR.push('camera-access');
+    }
+
+    @serializable()
+    public backgroundTint: RGBAColor = new RGBAColor(1,1,1,1);
+
+    public get background() {
+        return this.backgroundPlane;
+    }
+
+    private _preRender;
+
+    onEnable(): void {
+        this._preRender = this.preRender.bind(this);
+        this.context.pre_render_callbacks.push(this._preRender);
+
+        if (this.backgroundPlane) {
+            this.gameObject.add(this.backgroundPlane);
+            this.backgroundPlane.visible = false;
+        }
+    }
+
+    onDisable(): void {
+        this.context.pre_render_callbacks = this.context.pre_render_callbacks.filter(cb => cb !== this._preRender);
+
+        if (this.backgroundPlane)
+            this.gameObject.remove(this.backgroundPlane);
+    }
+
+    private backgroundPlane?: Mesh;
+    private threeTexture?: Texture;
+    private forceTextureInitialization = function() {
+        const material = new MeshBasicMaterial();
+        const geometry = new PlaneGeometry();
+        const scene = new Scene();
+        scene.add(new Mesh(geometry, material));
+        const camera = new PerspectiveCamera();
+    
+        return function forceTextureInitialization(renderer, texture) {
+          material.map = texture;
+          renderer.render(scene, camera);
+        };
+    }();
+
+    // TODO should only attach on session start, and detach on session end
+    private preRender() {
+        if (!this || !this.gameObject) return;
+
+        const xr = this.context.renderer.xr;
+        const frame = xr.getFrame();
+
+        if (frame) {
+            
+            // We're generating a new texture here, and force three to initialize it
+            // from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
+            if (!this.threeTexture && this.context.renderer) {
+                this.threeTexture = new Texture();
+                this.forceTextureInitialization(this.context.renderer, this.threeTexture);
+            }
+
+            // simple mesh and fullscreen shader to display the camera texture
+            // from three: WebGLBackground
+            if (this.backgroundPlane === undefined) {
+                this.backgroundPlane = makeFullscreenPlane(this.backgroundTint);
+                this.gameObject.add(this.backgroundPlane);
+            }
+
+            // WebXR Raw Camera Access - 
+            // we composite the camera texture into the scene background by rendering it first.
+            this.updateFromFrame(frame);
+        }
+
+        /*
+        if (this.planeMesh) {
+            this.planeMesh.visible = frame != null;
+        }
+        */
+    }
+
+    onBeforeRender(frame: XRFrame | null) {
+        this.updateFromFrame(frame);
+    }
+
+    updateFromFrame(frame: XRFrame | null) {
+        if (!frame) return;
+
+        // https://chromium.googlesource.com/chromium/src/+/7c5ac3c0f95b97cf12be95a5c1c0a8ff163246d8/third_party/webxr_test_pages/webxr-samples/proposals/camera-access-barebones.html
+        const pose = frame.getViewerPose(this.context.renderer.xr.getReferenceSpace()!);
+        if (pose) { 
+            for( const view of pose.views) {
+                // @ts-ignore
+                if ('camera' in view && view.camera) {
+                    const xrManager = this.context.renderer.xr;
+                    let binding = xrManager.getBinding();
+                    // not sure how / why this can be null, but we can recreate it here
+                    if (!binding) binding = new XRWebGLBinding( frame.session, this.context.renderer.getContext() );
+                    
+                    if (binding) {
+                        let glImage: WebGLTexture | null = null;
+                        if ('getCameraImage' in binding) {
+                            // @ts-ignore
+                            glImage = binding.getCameraImage(view.camera);
+
+                            // discussion on exactly this:
+                            // https://discourse.threejs.org/t/using-a-webgltexture-as-texture-for-three-js/46245/8
+                            // HACK from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
+                            const texProps = this.context.renderer.properties.get(this.threeTexture);
+                            texProps.__webglTexture = glImage;
+
+                            if (this.backgroundPlane) {
+                                //@ts-ignore
+                                this.backgroundPlane.setTexture(this.threeTexture);
+                                this.backgroundPlane.visible = true;
+                            }
+
+                            // TODO this would be a lot better but currently
+                            // setting color space doesn't work.
+                            // Plus we need to understand how we can supply a custom shader in
+                            // this case.
+                            /*
+                            if (this.threeTexture) {
+                                this.context.scene.background = this.threeTexture;
+                                this.threeTexture.colorSpace = NoColorSpace;
+                            }
+                            */
+                        }
+                    }
+                    else {
+                        // console.error(view.camera, xrManager)
+                    }
+                }
+                else {
+                    // console.error("NO CAMERA IN VIEW")
+                }
+            }
+        }
+        else {
+            // console.error(this.context.renderer.xr.getReferenceSpace(), frame);
+        }
+    }
+}
+// TODO tint could be an uniform
+const backgroundFragment: string = /* glsl */`
+uniform sampler2D t2D;
+
+varying vec2 vUv;
+
+void main() {
+
+    vec4 texColor = texture2D( t2D, vUv );
+    texColor.w = 1.0;
+
+    // inline sRGB decode
+    texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );
+
+    gl_FragColor = texColor * <backgroundTint>;
+
+    #include <tonemapping_fragment>
+    #include <encodings_fragment>
+
+}
+`;
+
+// not sure where we want to move this and in which form is best (extends Object3D?)
+export function makeFullscreenPlane(tint: RGBAColor ) {
+    const replacementTint = "vec4(" + tint.r.toFixed(3) + "," + tint.g.toFixed(3) + "," + tint.b.toFixed(3) + "," + tint.a.toFixed(3) + ")";
+    console.log(replacementTint);
+    const planeMesh = new Mesh(
+        new PlaneGeometry(2, 2),
+        // @ts-ignore
+        new ShaderMaterial({
+            name: 'BackgroundMaterial',
+            uniforms: UniformsUtils.clone( ShaderLib.background.uniforms ),
+            vertexShader: ShaderLib.background.vertexShader,
+            fragmentShader: backgroundFragment.replace("<backgroundTint>", replacementTint), // ShaderLib.background.fragmentShader,
+            side: DoubleSide,
+            depthTest: false,
+            depthWrite: false,
+            fog: false
+        })
+    );
+
+    planeMesh.geometry.deleteAttribute( 'normal' );
+
+    // add "map" material property so the renderer can evaluate it like for built-in materials
+    Object.defineProperty( planeMesh.material, 'map', {
+        get: function () {
+            return this.threeTexture;
+        }
+    } );
+
+    // Option 1: add the planeMesh to our scene for rendering.
+    // This is useful for applying custom shader effects on the background (instead of using the system composite)
+    planeMesh.renderOrder = -10000; // render first
+    planeMesh.layers.disableAll();
+    planeMesh.layers.enable(2); // ignore raycasts
+    planeMesh.frustumCulled = false;
+
+    // should be a class, for now lets just define a method for the weird way the texture needs to be set
+    // @ts-ignore
+    planeMesh.setTexture = function(texture) {
+        planeMesh.material.uniforms.t2D.value = texture;
+    }
+
+    return planeMesh;
 }