@needle-tools/engine 2.24.0-pre

package/src/engine-components/RendererLightmap.ts

@@ -0,0 +1,152 @@
+import { Behaviour, GameObject } from "./Component";
+import * as THREE from "three";
+import { Texture } from "three";
+import { Context, OnBeforeRenderCallback } from "../engine/engine_setup";
+
+// this component is automatically added by the Renderer if the object has lightmap uvs AND we have a lightmap
+// for multimaterial objects GLTF exports a "Group" with the renderer component
+// and every child mesh is a material from unity
+export class RendererLightmap {
+
+    get lightmap(): Texture | null {
+        return this.lightmapTexture;
+    }
+    set lightmap(tex: Texture | null) {
+        if (tex !== this.lightmapTexture) {
+            this.lightmapTexture = tex;
+            this.setupLightmap();
+        }
+    }
+
+    lightmapIndex: number = -1;
+    lightmapScaleOffset: THREE.Vector4 = new THREE.Vector4(1, 1, 0, 0);
+
+    private context: Context;
+    private gameObject: GameObject;
+    private lightmapTexture: THREE.Texture | null = null;
+    private lightmapScaleOffsetUniform = { value: new THREE.Vector4(1, 1, 0, 0) };
+    private lightmapUniform: { value: THREE.Texture | null } = { value: null };
+
+    private beforeRenderCallback?: OnBeforeRenderCallback;
+
+    constructor(gameObject: GameObject, context: Context) {
+        this.gameObject = gameObject;
+        this.context = context;
+    }
+
+    init(lightmapIndex: number, lightmapScaleOffset: THREE.Vector4, lightmapTexture: THREE.Texture, debug: boolean = false) {
+        console.assert(this.gameObject !== undefined && this.gameObject !== null, "Missing gameobject", this);
+
+        this.lightmapIndex = lightmapIndex;
+        if (this.lightmapIndex < 0) return;
+        this.lightmapScaleOffset = lightmapScaleOffset;
+        this.lightmapTexture = lightmapTexture;
+
+        const debugLightmaps = debug;
+        if (debugLightmaps) this.setLightmapDebugMaterial();
+        this.setupLightmap();
+    }
+
+    bindOnBeforeRender() {
+        this.beforeRenderCallback = this.onBeforeRenderThreeComplete.bind(this);
+        this.context.addBeforeRenderListener(this.gameObject, this.beforeRenderCallback);
+        // this.gameObject.onBeforeRender = this.onBeforeRenderThreeComplete.bind(this);
+    }
+
+    private onBeforeRenderThreeComplete(_renderer, _scene, _camera, _geometry, material, _group) {
+        this.onBeforeRenderThree(material);
+    }
+
+    private setupLightmap() {
+
+        if (this.gameObject.type === "Object3D") {
+            // console.warn("Can not add lightmap. Is this object missing a renderer?");
+            return;
+        }
+
+        if (this.gameObject.type === "Group") {
+            console.warn("Lightmap on multimaterial object is not supported yet... please ask kindly for implementation.");
+            return;
+        }
+
+        console.assert(this.gameObject.type === "Mesh", "Lightmap only works on meshes", this);
+
+
+        const mesh = this.gameObject as unknown as THREE.Mesh;
+        // TODO: ensure uv2 exists
+        if (!mesh.geometry.getAttribute("uv2"))
+            mesh.geometry.setAttribute("uv2", mesh.geometry.getAttribute("uv"));
+
+        const mat = this.gameObject["material"].clone();
+        this.gameObject["material"] = mat;
+
+        this.gameObject["material"].onBeforeCompile = (shader, _) => {
+            shader.uniforms.lightmap = this.lightmapUniform;
+            shader.uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
+        };
+
+
+        if (this.lightmapIndex >= 0) {
+            const lightmap = this.lightmapTexture;
+            const mat = this.gameObject["material"];
+            if (mat && lightmap) {
+                if (!mat.uniforms) mat.uniforms = {};
+                mat.lightMap = lightmap;
+                mat.uniforms.lightmap = { value: lightmap };
+            }
+        }
+    }
+
+    onBeforeRenderThree(material: THREE.Material) {
+
+        const uniforms = material["uniforms"];
+        if (uniforms && uniforms.lightmap) {
+            this.lightmapScaleOffsetUniform.value = this.lightmapScaleOffset;
+            this.lightmapUniform.value = this.lightmapTexture;
+            uniforms.lightmap = this.lightmapUniform;
+            uniforms.lightmapScaleOffset = this.lightmapScaleOffsetUniform;
+        }
+    }
+
+    private setLightmapDebugMaterial() {
+
+        // debug lightmaps
+        this.gameObject["material"] = new THREE.ShaderMaterial({
+            vertexShader: `
+                attribute vec2 uv2;
+                varying vec2 vUv2;
+                void main()
+                {
+                    vUv2 = uv2;
+                    gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
+                }
+                `,
+            fragmentShader: `
+                uniform sampler2D lightmap;
+                uniform float lightMapIntensity;
+                uniform vec4 lightmapScaleOffset;
+                varying vec2 vUv2;
+
+                // took from threejs 05fc79cd52b79e8c3e8dec1e7dca72c5c39983a4
+                vec4 conv_sRGBToLinear( in vec4 value ) {
+                    return vec4( mix( pow( value.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), value.rgb * 0.0773993808, vec3( lessThanEqual( value.rgb, vec3( 0.04045 ) ) ) ), value.a );
+                }
+
+                void main() {
+                    vec2 lUv = vUv2.xy * lightmapScaleOffset.xy + vec2(lightmapScaleOffset.z, (1. - (lightmapScaleOffset.y + lightmapScaleOffset.w)));
+                    
+                    vec4 lightMapTexel = texture2D( lightmap, lUv);
+                    // The range of RGBM lightmaps goes from 0 to 34.49 (5^2.2) in linear space, and from 0 to 5 in gamma space.
+                    //lightMapTexel.rgb *= lightMapTexel.a * 8.; // no idea where that "8" comes from... heuristically derived
+                    //lightMapTexel.a = 1.;
+                    //lightMapTexel = conv_sRGBToLinear(lightMapTexel);
+                    // lightMapTexel.rgb = vec3(1.);
+
+                    // gl_FragColor = vec4(vUv2.xy, 0, 1);
+                    gl_FragColor = lightMapTexel;
+                }
+                `,
+            defines: { USE_LIGHTMAP: '' }
+        });
+    }
+}

package/src/engine-components/RigidBody.ts

@@ -0,0 +1,177 @@
+import { Behaviour } from "./Component";
+import { BodyOptions } from '../engine/engine_physics'
+import * as CANNON from 'cannon-es'
+import * as THREE from 'three'
+import { getWorldPosition } from "../engine/engine_three_utils";
+import { onChange, RevocableProxy } from "./ui/Utils";
+import { serializeable } from "../engine/engine_serialization_decorator";
+
+export class Rigidbody extends Behaviour {
+
+    @serializeable()
+    mass: number = 1;
+    
+    @serializeable()
+    public set isKinematic(kinematic: boolean) {
+        this._isKinematic = kinematic;
+        if (this._body) {
+            this._body.type = kinematic ? CANNON.Body.KINEMATIC : CANNON.Body.DYNAMIC;
+        }
+    }
+
+    public get isKinematic(): boolean {
+        return this._isKinematic;
+    }
+    private _isKinematic: boolean = false;
+
+    @serializeable()
+    drag: number = 0;
+    @serializeable()
+    angularDrag: number = 1;
+    @serializeable()
+    detectCollision: boolean = true;
+    @serializeable()
+    sleepThreshold: number = 0.01;
+
+
+    get body(): CANNON.Body | null {
+        if (this._body === null) {
+            this.initialize();
+        }
+        return this._body;
+    }
+
+    private static tempPosition: THREE.Vector3 = new THREE.Vector3();
+    private _body: CANNON.Body | null = null;
+    private currentVelocity!: THREE.Vector3;
+    private _smoothedVelocity!: THREE.Vector3;
+    private lastWorldPosition!: THREE.Vector3;
+
+    private dirty: boolean = true;
+    private transformProxy? : RevocableProxy;
+
+    awake() {
+        this._body = null;
+        this.currentVelocity = new THREE.Vector3();
+        this._smoothedVelocity = new THREE.Vector3();
+        this.lastWorldPosition = getWorldPosition(this.gameObject).clone();
+    }
+
+    start(){
+        this.setBodyFromGameObject();
+    }
+
+    onEnable() {
+        if (this._body) {
+            this.context.physics.addBody(this.gameObject, this._body);
+        }
+        else
+            this.initialize();
+
+        if (this.body) {
+            this.body.wakeUp();
+            this.gameObject.updateWorldMatrix(true, false);
+            this.setBodyFromGameObject();
+        }
+
+        this.transformProxy = onChange(this.gameObject.matrix, "elements", () => {
+            if (!this.context.physics.isUpdating) {
+                this.dirty = true;
+            }
+        });
+    }
+
+    onDisable(): void {
+        this.transformProxy?.revoke();
+        if (this._body)
+            this.context.physics.removeBody(this.gameObject, this._body, false);
+    }
+
+    onDestroy(): void {
+        if (this._body)
+            this.context.physics.removeBody(this.gameObject, this._body);
+    }
+
+    earlyUpdate(){
+        if(this.dirty) this.setBodyFromGameObject();
+    }
+    onBeforeRender() {
+        this.updateVelocity();
+    }
+
+    initialize() {
+        if (this._body) return;
+        const options = new BodyOptions();
+        options.drag = this.drag;
+        options.angularDrag = this.angularDrag;
+        options.mass = this.mass;
+        options.kinematic = this.isKinematic;
+        options.physicsEvents = this.detectCollision;
+        options.sleepThreshold = this.sleepThreshold;
+        this._body = this.context.physics.createBody(this.gameObject, options);
+    }
+
+
+    public wakeUp() {
+        this.body?.wakeUp();
+    }
+
+    public applyForce(vec: THREE.Vector3, rel: THREE.Vector3 | undefined) {
+        this.body?.applyForce(new CANNON.Vec3(vec.x, vec.y, vec.z), rel ? new CANNON.Vec3(rel.x, rel.y, rel.z) : undefined);
+    }
+
+    public getVelocity(): THREE.Vector3 {
+        if (this.body)
+            return Rigidbody.tempPosition.set(this.body?.velocity.x, this.body?.velocity.y, this.body?.velocity.z);
+        return Rigidbody.tempPosition.set(0, 0, 0);
+    }
+
+    public setVelocity(x: number, y: number, z: number) {
+        if (!this.body) return;
+        this.body.velocity.x = x / this.context.time.deltaTime;
+        this.body.velocity.y = y / this.context.time.deltaTime;
+        this.body.velocity.z = z / this.context.time.deltaTime;
+    }
+
+    public get smoothedVelocity() { return this._smoothedVelocity; }
+
+    public setAngularVelocity(x: number, y: number, z: number) {
+        if (!this.body) return;
+        this.body.angularVelocity.x = x / this.context.time.deltaTime;
+        this.body.angularVelocity.y = y / this.context.time.deltaTime;
+        this.body.angularVelocity.z = z / this.context.time.deltaTime;
+    }
+
+    private static copyVector3: THREE.Vector3 = new THREE.Vector3();
+
+
+    public setBodyFromGameObject(velocity: THREE.Vector3 | null | { x: number, y: number, z: number } = null) {
+        this.dirty = false;
+        if (this.body && this.gameObject && !this.destroyed) {
+            const wp = this.worldPosition;
+            this.body.position.set(wp.x, wp.y, wp.z);
+            const wr = this.worldQuaternion;
+            this.body.quaternion.set(wr.x, wr.y, wr.z, wr.w);
+
+            if (velocity) {
+                Rigidbody.copyVector3.set(velocity.x, velocity.y, velocity.z);
+                this._smoothedVelocity.lerp(Rigidbody.copyVector3, this.context.time.deltaTime / .1);
+                const sm = this._smoothedVelocity;
+                this.body.velocity.x = sm.x;
+                this.body.velocity.y = sm.y;
+                this.body.velocity.z = sm.z;
+            }
+        }
+    }
+
+
+    private updateVelocity() {
+        if (!this.currentVelocity) return;
+        if (this.body && this.gameObject) {
+            const wp = getWorldPosition(this.gameObject);
+            this.currentVelocity.subVectors(wp, this.lastWorldPosition);
+            this.lastWorldPosition.copy(wp);
+            this._smoothedVelocity.lerp(this.currentVelocity, this.context.time.deltaTime / .1);
+        }
+    }
+}

package/src/engine-components/ScreenCapture.ts

@@ -0,0 +1,47 @@
+import { Behaviour, GameObject } from "./Component";
+import { VideoPlayer } from "./VideoPlayer";
+
+export class ScreenCapture extends Behaviour {
+
+    streamOnAwake: boolean = true;
+
+    start() {
+        if (this.streamOnAwake)
+            this.open();
+    }
+
+    private requestOpen: boolean = false;
+    private captureStream: MediaStream | null = null;
+
+    async open(force: boolean = false) {
+        if (this.captureStream && !force) return;
+        this.close();
+        this.requestOpen = true;
+        try {
+            const player = GameObject.getComponentInChildren(this.gameObject, VideoPlayer);
+            if (player) {
+                const displayMediaOptions: DisplayMediaStreamConstraints = {
+                    video: true,
+                    audio: false
+                };
+                this.captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+                if (this.captureStream && this.requestOpen) {
+                    //@ts-ignore
+                    player.clip = this.captureStream;
+                    player.create(true);
+                }
+            }
+        } catch (err) {
+            console.error("Error: " + err);
+        }
+    }
+
+    close() {
+        this.requestOpen = false;
+        if (this.captureStream) {
+            for (const cap of this.captureStream.getTracks())
+                cap.stop();
+            this.captureStream = null;
+        }
+    }
+}

package/src/engine-components/ShadowCatcher.ts

@@ -0,0 +1,93 @@
+import { Behaviour, GameObject } from "./Component";
+import { RGBAColor } from "./js-extensions/RGBAColor";
+import { Renderer } from "./Renderer";
+import { ShadowMaterial, AdditiveBlending, Material } from "three";
+import { serializeable } from "../engine/engine_serialization_decorator";
+
+enum ShadowMode {
+    ShadowMask = 0,
+    Additive = 1,
+}
+
+export class ShadowCatcher extends Behaviour {
+
+    //@type Needle.Engine.ShadowCatcher.Mode
+    @serializeable()
+    mode: ShadowMode = ShadowMode.ShadowMask;
+
+    //@type UnityEngine.Color
+    @serializeable(RGBAColor)
+    shadowColor: RGBAColor = new RGBAColor(0, 0, 0, 1);
+
+    awake() {
+
+        switch (this.mode) {
+            case ShadowMode.ShadowMask:
+                this.applyShadowMaterial();
+                break;
+            case ShadowMode.Additive:
+                this.applyLightBlendMaterial();
+                break;
+        }
+
+    }
+
+    // Custom blending, diffuse-only lighting blended onto the scene additively.
+    // Works great for Point Lights and spot lights, 
+    // doesn't work for directional lights (since they're lighting up everything else).
+    // Works even better with an additional black-ish gradient to darken parts of the AR scene
+    // so that lights become more visible on bright surfaces.
+    applyLightBlendMaterial() {
+        const renderer = GameObject.getComponent(this.gameObject, Renderer);
+        if (renderer) {
+            const material = renderer.material;
+            material.blending = AdditiveBlending;
+            this.applyMaterialOptions(material);
+            material.onBeforeCompile = (shader) => {
+                // see https://github.com/mrdoob/three.js/blob/dev/src/renderers/shaders/ShaderLib/meshphysical.glsl.js#L181
+                shader.fragmentShader = shader.fragmentShader.replace("vec3 outgoingLight = totalDiffuse + totalSpecular + totalEmissiveRadiance;",
+                    `vec3 outgoingLight = totalDiffuse + totalSpecular + totalEmissiveRadiance;
+                
+                // diffuse-only lighting with overdrive to somewhat compensate
+                // for the loss of indirect lighting and to make it more visible.
+                vec3 direct = reflectedLight.directDiffuse * 3.;
+                float max = max(direct.r, max(direct.g, direct.b));
+                
+                // early out - we're simply returning direct lighting and some alpha based on it so it can 
+                // be blended onto the scene.
+                gl_FragColor = vec4(direct, max);
+                return;
+                `);
+            }
+        }
+    }
+
+    // THREE.ShadowMaterial: only does a mask; shadowed areas are fully black.
+    // doesn't take light attenuation into account.
+    // works great for Directional Lights.
+    applyShadowMaterial() {
+        const renderer = GameObject.getComponent(this.gameObject, Renderer);
+        if (renderer) {
+            if (renderer.material?.type !== "ShadowMaterial") {
+                const material = new ShadowMaterial();
+                material.color = this.shadowColor;
+                material.opacity = this.shadowColor.alpha;
+                this.applyMaterialOptions(material);
+                renderer.material = material;
+            }
+            else {
+                const material = renderer.material as ShadowMaterial;
+                material.color = this.shadowColor;
+                material.opacity = this.shadowColor.alpha;
+                this.applyMaterialOptions(material);
+            }
+        }
+    }
+
+    private applyMaterialOptions(material: Material) {
+        if (material) {
+            material.depthWrite = false;
+            material.stencilWrite = false;
+        }
+    }
+}

package/src/engine-components/Skybox.ts

@@ -0,0 +1,116 @@
+import { serializeable } from "../engine/engine_serialization_decorator";
+import { Behaviour, GameObject } from "./Component";
+import { RGBELoader } from 'three/examples/jsm/loaders/RGBELoader.js';
+import { EXRLoader } from "three/examples/jsm/loaders/EXRLoader";
+import { EquirectangularRefractionMapping, Texture } from "three"
+import { syncField } from "../engine/engine_networking_auto";
+
+
+export class RemoteSkybox extends Behaviour {
+
+    @syncField("setSkybox")
+    @serializeable()
+    url?: string;
+
+    @serializeable()
+    allowDrop : boolean = true;
+
+    private _loader?: RGBELoader | EXRLoader;
+    private _prevLoadedEnvironment?: Texture;
+    private _prevEnvironment: Texture | null = null;
+    private _prevBackground: any = null;
+
+    onEnable() {
+        this.setSkybox(this.url);
+        this.registerDropEvents();
+    }
+
+    onDisable() {
+        if (this.context.scene.environment === this._prevLoadedEnvironment) {
+            if (!this.context.isInXR) {
+                this.context.scene.environment = this._prevEnvironment;
+                this.context.scene.background = this._prevBackground;
+            }
+            this._prevLoadedEnvironment = undefined;
+        }
+        this.unregisterDropEvents();
+    }
+
+    async setSkybox(url: string | undefined | null) {
+        if (!url) return;
+        if (!url?.endsWith(".hdr") && !url.endsWith(".exr")) {
+            console.warn("Potentially invalid skybox url", this.url, "on", this.name);
+        }
+
+        if (!this._loader) {
+            const isEXR = url.endsWith(".exr");
+            this._loader = isEXR ? new EXRLoader() : new RGBELoader();
+        }
+        const envMap = await this._loader.loadAsync(url);
+        if (!envMap) return;
+        if (!this.enabled) return;
+        this.url = url;
+        envMap.mapping = EquirectangularRefractionMapping;
+        this._prevBackground = this.context.scene.background;
+        this._prevEnvironment = this.context.scene.environment;
+        console.log("Set skybox", this.url);
+        this.context.scene.environment = envMap;
+        this.context.scene.background = envMap;
+        this._prevLoadedEnvironment = envMap;
+        const nameIndex = url.lastIndexOf("/");
+        envMap.name = url.substring(nameIndex >= 0 ? nameIndex + 1 : 0);
+    }
+
+
+    private dragOverEvent?: any;
+    private dropEvent?: any;
+
+    private registerDropEvents() {
+        if (this.dragOverEvent) return;
+
+        this.dragOverEvent = (e: DragEvent) => {
+            if(!this.allowDrop) return;
+            if (!e.dataTransfer) return;
+            for (const type of e.dataTransfer.types) {
+                // in ondragover we dont get access to the content
+                // but if we have a uri list we can assume
+                // someone is maybe dragging a image file
+                // so we want to capture this
+                if (type === "text/uri-list") {
+                    e.preventDefault();
+                }
+            }
+        };
+
+        this.dropEvent = (e: DragEvent) => {
+            if(!this.allowDrop) return;
+            e.preventDefault();
+            if (!e.dataTransfer) return;
+            for (const type of e.dataTransfer.types) {
+                if (type === "text/uri-list") {
+                    const url = e.dataTransfer.getData(type);
+                    console.log(type, url);
+                    let name = new RegExp(/polyhaven.com\/asset_img\/.+?\/(?<name>.+)\.png/).exec(url)?.groups?.name;
+                    if (!name) {
+                        name = new RegExp(/polyhaven\.com\/a\/(?<name>.+)/).exec(url)?.groups?.name;
+                    }
+                    console.log(name);
+                    if (name) {
+                        const envurl = "https://dl.polyhaven.org/file/ph-assets/HDRIs/exr/1k/" + name + "_1k.exr";
+                        this.setSkybox(envurl);
+                    }
+                    else console.warn("Could not resolve skybox name from dropped url", url);
+                }
+            }
+        };
+
+        this.context.domElement.addEventListener("dragover", this.dragOverEvent);
+        this.context.domElement.addEventListener("drop", this.dropEvent);
+    }
+
+    private unregisterDropEvents() {
+        if (!this.dragOverEvent) return;
+        this.context.domElement.removeEventListener("dragover", this.dragOverEvent);
+        this.context.domElement.removeEventListener("drop", this.dropEvent);
+    }
+}

package/src/engine-components/SmoothFollow.ts

@@ -0,0 +1,55 @@
+import { Camera } from "./Camera";
+import { Behaviour, GameObject } from "./Component";
+import * as utils from "../engine/engine_three_utils";
+import * as THREE from "three";
+import { Mathf } from "../engine/engine_math";
+import { serializeable } from "../engine/engine_serialization_decorator";
+import { Object3D } from "three";
+
+export class SmoothFollow extends Behaviour {
+
+    @serializeable(Object3D)
+    target: THREE.Object3D | null = null;
+
+    followFactor = .1;
+    rotateFactor = .1;
+
+    flipForward:boolean = false;
+
+    private static _invertForward : THREE.Quaternion = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);
+    private _firstUpdate = true;
+
+
+    onEnable(): void {
+        const cam = GameObject.getComponentInChildren(this.gameObject, Camera) as Camera;
+        if (cam && cam.cam) {
+            cam.gameObject.position.set(0, 0, 0);
+            cam.cam.position.set(0, 0, 0);
+            cam.gameObject.quaternion.identity();
+            cam.cam.quaternion.setFromEuler(new THREE.Euler(0, Math.PI, 0));
+        }
+    }
+
+    onBeforeRender(): void {
+        if (this.followFactor <= 0) return;
+        this.updateNow(false);
+    }
+
+    updateNow(hard: boolean) {
+        if (!this.target || this.target === this.gameObject) return;
+        if (this.followFactor > 0) {
+            const wp = utils.getWorldPosition(this.target);
+            const fpos = this._firstUpdate || hard ? 1 : Mathf.clamp01(this.context.time.deltaTime * this.followFactor);
+            this.worldPosition = this.worldPosition.lerp(wp, fpos);
+        }
+        if (this.rotateFactor > 0) {
+            const wr = utils.getWorldQuaternion(this.target);
+            if(this.flipForward){
+                wr.multiply(SmoothFollow._invertForward);
+            }
+            const frot = this._firstUpdate || hard ? 1 : Mathf.clamp01(this.context.time.deltaTime * this.rotateFactor);
+            this.worldQuaternion = this.worldQuaternion.slerp(wr, frot);
+        }
+        this._firstUpdate = false;
+    }
+}