@needle-tools/engine 2.32.0-pre → 2.34.0-pre

package/src/engine-components/SyncedCamera.ts

@@ -7,7 +7,7 @@ import { Builder } from "flatbuffers";
 import { SyncedCameraModel } from "../engine-schemes/synced-camera-model";
 import { Vec3 } from "../engine-schemes/vec3";
 import { registerType } from "../engine-schemes/schemes";
-import { InstancingUtil } from "./Renderer";
+import { InstancingUtil } from "../engine/engine_instancing";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { Object3D } from "three";
 import { AvatarMarker } from "./WebXRAvatar";

package/src/engine-components/SyncedTransform.ts

@@ -4,7 +4,7 @@ import { Behaviour, GameObject } from "./Component";
 import { Rigidbody } from "./RigidBody";
 import * as utils from "../engine/engine_utils"
 import { sendDestroyed } from '../engine/engine_networking_instantiate';
-import { InstancingUtil } from './Renderer';
+import { InstancingUtil } from "../engine/engine_instancing";
 import { SyncedTransformModel } from '../engine-schemes/synced-transform-model';
 import * as flatbuffers from "flatbuffers";
 import { Transform } from '../engine-schemes/transform';

package/src/engine-components/VideoPlayer.ts

@@ -1,14 +1,24 @@
-import { Behaviour } from "./Component";
+import { Behaviour, GameObject } from "./Component";
 import * as THREE from "three";
 import { Material } from "material/Material";
 import { serializeable } from "../engine/engine_serialization_decorator";
-import { LinearFilter, Object3D, Texture } from "three";
+import { LinearFilter, Mesh, Object3D, RawShaderMaterial, ShaderMaterial, Texture, TextureLoader, Vector2, Vector4, VideoTexture } from "three";
 import { awaitInput } from "../engine/engine_input_utils";
 import { getParam } from "../engine/engine_utils";
 import { Renderer } from "./Renderer";
+import { getWorldScale } from "../engine/engine_three_utils";
+import { ObjectUtils, PrimitiveType } from "../engine/engine_create_objects";
+import { Context } from "../engine/engine_setup";
 
 const debug = getParam("debugvideo");
 
+
+export enum AspectMode {
+    None = 0,
+    AdjustHeight = 1,
+    AdjustWidth = 2,
+}
+
 export enum VideoSource {
     /// <summary>
     ///   <para>Use the current clip as the video content source.</para>
@@ -43,6 +53,9 @@ export class VideoPlayer extends Behaviour {
     @serializeable()
     playOnEnable?: boolean;
 
+    @serializeable()
+    aspectMode: AspectMode = AspectMode.None;
+
     @serializeable()
     private renderMode?: VideoRenderMode;
 
@@ -110,6 +123,10 @@ export class VideoPlayer extends Behaviour {
         }
     }
 
+    get videoTexture() {
+        return this._videoTexture;
+    }
+
     private _crossOrigin: string | null = "anonymous";
 
     private audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.AudioSource;
@@ -119,7 +136,7 @@ export class VideoPlayer extends Behaviour {
     private url?: string | null = null;
 
     private videoElement: HTMLVideoElement | null = null;
-    private videoTexture: THREE.VideoTexture | null = null;
+    private _videoTexture: THREE.VideoTexture | null = null;
     private videoMaterial: Material | null = null;
 
     private _isPlaying: boolean = false;
@@ -134,6 +151,7 @@ export class VideoPlayer extends Behaviour {
             this.videoElement.srcObject = video;
             if (this._isPlaying)
                 this.videoElement.play();
+            this.updateAspect();
         }
     }
 
@@ -146,8 +164,10 @@ export class VideoPlayer extends Behaviour {
         if (!this.videoElement) this.create(true);
         else {
             this.videoElement.src = url;
-            if (this._isPlaying)
-                this.videoElement.play();
+            if (this._isPlaying) {
+                this.stop();
+                this.play();
+            }
         }
     }
 
@@ -171,6 +191,10 @@ export class VideoPlayer extends Behaviour {
         if (this.playOnEnable === true) {
             this.handleBeginPlaying(true);
         }
+        if (this.screenspace) {
+            this._overlay?.start();
+        }
+        else this._overlay?.stop();
     }
 
     onDisable(): void {
@@ -182,9 +206,9 @@ export class VideoPlayer extends Behaviour {
             this.videoElement.parentElement?.removeChild(this.videoElement);
             this.videoElement = null;
         }
-        if (this.videoTexture) {
-            this.videoTexture.dispose();
-            this.videoTexture = null;
+        if (this._videoTexture) {
+            this._videoTexture.dispose();
+            this._videoTexture = null;
         }
     }
 
@@ -197,6 +221,14 @@ export class VideoPlayer extends Behaviour {
             this.updateVideoElementSettings();
         });
         this._targetObjects = [];
+
+        if (getParam("videoscreenspace")) {
+            window.addEventListener("keydown", evt => {
+                if (evt.key === "f") {
+                    this.screenspace = !this.screenspace;
+                }
+            });
+        }
     }
 
     play() {
@@ -245,24 +277,46 @@ export class VideoPlayer extends Behaviour {
             // hide it because otherwise it would overlay the website with default css
             this.updateVideoElementStyles();
         }
-        if (typeof src === "string")
+        if (typeof src === "string") {
             this.videoElement.src = src;
+            const str = this.videoElement["captureStream"]?.call(this.videoElement);
+            this.clip = str;
+        }
         else
             this.videoElement.srcObject = src;
 
-        if (!this.videoTexture)
-            this.videoTexture = new THREE.VideoTexture(this.videoElement);
-        this.videoTexture.flipY = false;
-        this.videoTexture.encoding = THREE.sRGBEncoding;
+
+        if (!this._videoTexture)
+            this._videoTexture = new THREE.VideoTexture(this.videoElement);
+        this._videoTexture.flipY = false;
+        this._videoTexture.encoding = THREE.sRGBEncoding;
         this.handleBeginPlaying(playAutomatically);
+        console.log(this);
     }
 
-    private _targetObjects: Array<Object3D>;
+    updateAspect() {
+        if (this.aspectMode === AspectMode.None) return;
+        this.startCoroutine(this.updateAspectImpl());
+    }
+
+    private _overlay: VideoOverlay | null = null;
+
+    get screenspace(): boolean {
+        return this._overlay?.enabled ?? false;
+    }
 
-    getTargetObjects(): Array<Object3D> {
-        return Array.from(this._targetObjects);
+    set screenspace(val: boolean) {
+        if (val) {
+            if (!this._videoTexture) return;
+            if (!this._overlay) this._overlay = new VideoOverlay(this.context);
+            this._overlay.add(this._videoTexture);
+        }
+        else this._overlay?.remove(this._videoTexture);
+        if (this._overlay) this._overlay.enabled = val;
     }
 
+    private _targetObjects: Array<Object3D>;
+
     private createVideoElement(): HTMLVideoElement {
         const video = document.createElement("video") as HTMLVideoElement;
         if (this._crossOrigin)
@@ -282,6 +336,7 @@ export class VideoPlayer extends Behaviour {
         switch (this.renderMode) {
             case VideoRenderMode.MaterialOverride:
                 target = this.targetMaterialRenderer?.gameObject;
+                if (!target) target = GameObject.getComponent(this.gameObject, Renderer)?.gameObject;
                 break;
             case VideoRenderMode.RenderTexture:
                 console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
@@ -289,25 +344,25 @@ export class VideoPlayer extends Behaviour {
         }
 
         if (!target) {
-            console.error("Missing target for video material renderer", this);
+            console.error("Missing target for video material renderer", this.name, VideoRenderMode[this.renderMode!], this);
             return;
         }
         const mat = target["material"];
         if (mat) {
             this._targetObjects.push(target);
-            
+
             if (mat !== this.videoMaterial) {
                 this.videoMaterial = mat.clone();
                 target["material"] = this.videoMaterial;
             }
 
             if (!this.targetMaterialProperty) {
-                (this.videoMaterial as any).map = this.videoTexture;
+                (this.videoMaterial as any).map = this._videoTexture;
             }
             else {
                 switch (this.targetMaterialProperty) {
                     default:
-                        (this.videoMaterial as any).map = this.videoTexture;
+                        (this.videoMaterial as any).map = this._videoTexture;
                         break;
                     // doesnt render:
                     // case "emissiveTexture":
@@ -348,61 +403,359 @@ export class VideoPlayer extends Behaviour {
         this.videoElement.style.userSelect = "none";
         this.videoElement.style.visibility = "hidden";
         this.videoElement.style.display = "none";
+        this.updateAspect();
+    }
+
+
+
+
+    private _updateAspectRoutineId: number = -1;
+    private *updateAspectImpl() {
+        const id = ++this._updateAspectRoutineId;
+        const lastAspect: number | undefined = undefined;
+        const stream = this.clip;
+        while (id === this._updateAspectRoutineId && this.aspectMode !== AspectMode.None && this.clip && stream === this.clip && this._isPlaying) {
+            if (!stream || typeof stream === "string") {
+                return;
+            }
+            let aspect: number | undefined = undefined;
+            for (const track of stream.getVideoTracks()) {
+                const settings = track.getSettings();
+                if (settings && settings.width && settings.height) {
+                    aspect = settings.width / settings.height;
+                    break;
+                }
+                // on firefox capture canvas stream works but looks like 
+                // the canvas stream track doesnt contain settings?!!?
+                else {
+                    aspect = this.context.renderer.domElement.clientWidth / this.context.renderer.domElement.clientHeight;
+                }
+            }
+            if (aspect === undefined) {
+                for (let i = 0; i < 10; i++)
+                    yield;
+                if (!this.isPlaying) break;
+                continue;
+            }
+            if (lastAspect === aspect) {
+                yield;
+                continue;
+            }
+            for (const obj of this._targetObjects) {
+                let worldAspect = 1;
+                if (obj.parent) {
+                    const parentScale = getWorldScale(obj.parent);
+                    worldAspect = parentScale.x / parentScale.y;
+                }
+                switch (this.aspectMode) {
+                    case AspectMode.AdjustHeight:
+                        obj.scale.y = 1 / aspect * obj.scale.x * worldAspect;
+                        break;
+                    case AspectMode.AdjustWidth:
+                        obj.scale.x = aspect * obj.scale.y * worldAspect;
+                        break;
+                }
+            }
+            for (let i = 0; i < 3; i++)
+                yield;
+        }
     }
 }
 
 
+class VideoOverlay {
 
+    readonly context: Context;
 
-// class VideoTexture extends Texture {
+    constructor(context: Context) {
+        this.context = context;
+        this._input = new VideoOverlayInput(this);
+    }
 
-//     get isVideoTexture() { return true; }
+    get enabled() {
+        return this._isInScreenspaceMode;
+    }
 
-// 	constructor( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy ) {
+    set enabled(val: boolean) {
+        if (val) this.start();
+        else this.stop();
+    }
 
-// 		super( video, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy );
 
-// 		this.minFilter = minFilter !== undefined ? minFilter : LinearFilter;
-// 		this.magFilter = magFilter !== undefined ? magFilter : LinearFilter;
+    add(video: VideoTexture) {
+        if (this._videos.indexOf(video) === -1) {
+            this._videos.push(video);
+        }
+    }
 
-// 		this.generateMipmaps = false;
+    remove(video: VideoTexture | null | undefined) {
+        if (!video) return;
+        const index = this._videos.indexOf(video);
+        if (index >= 0) {
+            this._videos.splice(index, 1);
+        }
+    }
 
-// 		const scope = this;
+    start() {
+        if (this._isInScreenspaceMode) return;
+        if (this._videos.length < 0) return;
+        const texture = this._videos[this._videos.length - 1];
+        if (!texture) return;
+
+        this._isInScreenspaceMode = true;
+        if (!this._screenspaceModeQuad) {
+            this._screenspaceModeQuad = ObjectUtils.createPrimitive(PrimitiveType.Quad, {
+                material: new ScreenspaceTexture(texture)
+            });
+            if (!this._screenspaceModeQuad) return;
+            this._screenspaceModeQuad.geometry.scale(2, 2, 2);
+        }
+        
+        const quad = this._screenspaceModeQuad;
+        this.context.scene.add(quad);
+        this.updateScreenspaceMaterialUniforms();
 
-// 		function updateVideo() {
+        const mat = quad.material as ScreenspaceTexture;
+        mat?.reset();
 
-// 			scope.needsUpdate = true;
-// 			video.requestVideoFrameCallback( updateVideo );
+        this._input?.enable(mat);
+    }
 
-// 		}
+    stop() {
+        this._isInScreenspaceMode = false;
+        if (this._screenspaceModeQuad) {
+            this._input?.disable();
+            this._screenspaceModeQuad.removeFromParent();
+        }
+    }
+
+    updateScreenspaceMaterialUniforms() {
+        const mat = this._screenspaceModeQuad?.material as ScreenspaceTexture;
+        if (!mat) return;
+        // mat.videoAspect = this.videoTexture?.image?.width / this.videoTexture?.image?.height;
+        mat.screenAspect = this.context.domElement.clientWidth / this.context.domElement.clientHeight;
+    }
+
+    private _videos: VideoTexture[] = [];
+    private _screenspaceModeQuad: Mesh | undefined;
+    private _isInScreenspaceMode: boolean = false;
+    private _input: VideoOverlayInput;
+}
+
+class VideoOverlayInput {
+
+    private _onResizeScreenFn?: () => void;
+    private _onKeyUpFn?: (e: KeyboardEvent) => void;
+    private _onMouseWheelFn?: (e: WheelEvent) => void;
+
+    private readonly context: Context;
+    private readonly overlay: VideoOverlay;
+
+    constructor(overlay: VideoOverlay) {
+        this.overlay = overlay;
+        this.context = overlay.context;
+    }
+
+    private _material?: ScreenspaceTexture;
+
+    enable(mat: ScreenspaceTexture) {
+        this._material = mat;
+
+        window.addEventListener("resize", this._onResizeScreenFn = () => {
+            this.overlay.updateScreenspaceMaterialUniforms();
+        });
+        window.addEventListener("keyup", this._onKeyUpFn = (args) => {
+            if (args.key === "Escape")
+                this.overlay.stop();
+        });
+
+        window.addEventListener("wheel", this._onMouseWheelFn = (args) => {
+            if (this.overlay.enabled) {
+                mat.zoom += args.deltaY * .0005;
+                args.preventDefault();
+            }
+        }, { passive: false });
+
+
+        const delta: Vector2 = new Vector2();
+
+        window.addEventListener("mousemove", (args: MouseEvent) => {
+            if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
+                const normalizedMovement = new Vector2(args.movementX, args.movementY);
+                normalizedMovement.x /= this.context.domElement.clientWidth;
+                normalizedMovement.y /= this.context.domElement.clientHeight;
+                delta.set(normalizedMovement.x, normalizedMovement.y);
+                delta.multiplyScalar(mat.zoom / -this.context.time.deltaTime * .01);
+                mat.offset = mat.offset.add(delta);
+            }
+        });
+
+        window.addEventListener("pointermove", (args: PointerEvent) => {
+            if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
+                const count = this.context.input.getTouchesPressedCount();
+                if (count === 1) {
+                    delta.set(args.movementX, args.movementY);
+                    delta.multiplyScalar(mat.zoom * -this.context.time.deltaTime * .05);
+                    mat.offset = mat.offset.add(delta);
+                }
+            }
+        });
 
-// 		if ( 'requestVideoFrameCallback' in video ) {
+        let lastTouchStartTime = 0;
+        window.addEventListener("touchstart", e => {
+            if (e.touches.length < 2) {
+                if (this.context.time.time - lastTouchStartTime < .3) {
+                    this.overlay.stop();
+                }
+                lastTouchStartTime = this.context.time.time;
+                return;
+            }
+            this._isPinching = true;
+            this._lastPinch = 0;
+        })
+        window.addEventListener("touchmove", e => {
+            if (!this._isPinching || !this._material) return;
+            const touch1 = e.touches[0];
+            const touch2 = e.touches[1];
+            const dx = touch1.clientX - touch2.clientX;
+            const dy = touch1.clientY - touch2.clientY;
+            const distance = Math.sqrt(dx * dx + dy * dy);
+            if (this._lastPinch !== 0) {
+                const delta = distance - this._lastPinch;
+                this._material.zoom -= delta * .004;
+            }
+            this._lastPinch = distance;
+        })
+        window.addEventListener("touchend", () => {
+            this._isPinching = false;
+        })
+    }
 
-// 			video.requestVideoFrameCallback( updateVideo );
+    private _isPinching: boolean = false;
+    private _lastPinch = 0;
 
-// 		}
+    disable() {
 
-// 	}
+        if (this._onResizeScreenFn) {
+            window.removeEventListener("resize", this._onResizeScreenFn);
+            this._onResizeScreenFn = undefined;
+        }
+        if (this._onKeyUpFn) {
+            window.removeEventListener("keyup", this._onKeyUpFn);
+            this._onKeyUpFn = undefined;
+        }
+        if (this._onMouseWheelFn) {
+            window.removeEventListener("wheel", this._onMouseWheelFn);
+            this._onMouseWheelFn = undefined;
+        }
+    }
 
-// 	// clone() {
+}
 
-// 	// 	return new this.constructor( this.image ).copy( this );
+class ScreenspaceTexture extends ShaderMaterial {
 
-// 	// }
+    set screenAspect(val: number) {
+        this.uniforms["screenAspect"].value = val;
+        this.needsUpdate = true;
+    }
 
-// 	update() {
+    set offset(vec: Vector2 | { x: number, y: number }) {
+        const val = this.uniforms["offsetScale"].value;
+        val.x = vec.x;
+        val.y = vec.y;
+        // console.log(val);
+        this.uniforms["offsetScale"].value = val;
+        this.needsUpdate = true;
+    }
 
-// 		const video = this.image;
-// 		const hasVideoFrameCallback = 'requestVideoFrameCallback' in video;
+    private readonly _offset: Vector2 = new Vector2();
+    get offset(): Vector2 {
+        const val = this.uniforms["offsetScale"].value;
+        this._offset.set(val.x, val.y);
+        return this._offset;
+    }
 
-// 		if ( hasVideoFrameCallback === false && video.readyState >= video.HAVE_CURRENT_DATA ) {
+    set zoom(val: number) {
+        const zoom = this.uniforms["offsetScale"].value;
+        if (val < .001) val = .001;
+        zoom.z = val;
+        // zoom.z = this.maxZoom - val;
+        // zoom.z /= this.maxZoom;
+        this.needsUpdate = true;
+    }
 
-// 			this.needsUpdate = true;
+    get zoom(): number {
+        return this.uniforms["offsetScale"].value.z;// * this.maxZoom;
+    }
+
+    reset() {
+        this.offset = this.offset.set(0, 0);
+        this.zoom = 1;
+        this.needsUpdate = true;
+    }
 
-// 		}
+    // maxZoom : number = 10
 
-// 	}
+    constructor(tex: Texture) {
+        super();
+
+        this.uniforms = {
+            map: { value: tex },
+            screenAspect: { value: 1 },
+            offsetScale: { value: new Vector4(0, 0, 1, 1) }
+        };
+
+        this.vertexShader = `
+        uniform sampler2D map;
+        uniform float screenAspect;
+        uniform vec4 offsetScale;
+        varying vec2 vUv;
+
+        void main() {
+
+            gl_Position = vec4( position , 1.0 );
+            vUv = uv;
+            vUv.y = 1. - vUv.y;
+
+            // fit into screen
+            ivec2 res = textureSize(map, 0);
+            float videoAspect = float(res.x) / float(res.y);
+            float aspect = videoAspect / screenAspect;
+            if(aspect >= 1.0) 
+            {
+                vUv.y = vUv.y * aspect;
+                float offset = (1. - aspect) * .5;
+                vUv.y = vUv.y + offset;
+            }
+            else
+            {
+                vUv.x = vUv.x / aspect;
+                float offset = (1. - 1. / aspect) * .5;
+                vUv.x = vUv.x + offset;
+            }
 
-// }
+            vUv.x -= .5;
+            vUv.y -= .5;
 
-// export { VideoTexture };
+            vUv.x *= offsetScale.z;
+            vUv.y *= offsetScale.z;
+            vUv.x += offsetScale.x;
+            vUv.y += offsetScale.y;
+
+            vUv.x += .5;
+            vUv.y += .5;
+        }
+
+        `
+        this.fragmentShader = `
+        uniform sampler2D map;
+        varying vec2 vUv;
+        void main() {
+            if(vUv.x < 0. || vUv.x > 1. || vUv.y < 0. || vUv.y > 1.)
+                gl_FragColor = vec4(0., 0., 0., 1.);
+            else
+                gl_FragColor = texture2D(map, vUv);
+        }
+        `
+    }
+}

package/src/engine-components/WebARSessionRoot.ts

@@ -2,7 +2,7 @@ import { Behaviour, GameObject } from "./Component";
 import * as THREE from 'three'
 import { Matrix4, Object3D, XRPose } from "three";
 import { WebAR, WebXR } from "./WebXR";
-import { InstancingUtil } from "./Renderer";
+import { InstancingUtil } from "../engine/engine_instancing";
 import { serializeable } from "../engine/engine_serialization_decorator";
 
 // https://github.com/takahirox/takahirox.github.io/blob/master/three.js.mmdeditor/examples/js/controls/DeviceOrientationControls.js

package/src/engine-components/WebXRAvatar.ts

@@ -11,7 +11,7 @@ import { VRUserState } from "./WebXRSync";
 import { getParam } from "../engine/engine_utils";
 import { serializeable } from "../engine/engine_serialization_decorator";
 import { ViewDevice } from "../engine/engine_playerview";
-import { InstancingUtil } from "./Renderer";
+import { InstancingUtil } from "../engine/engine_instancing";
 
 export const debug = getParam("debugavatar");
 

package/src/engine-components/WebXRController.ts

@@ -11,7 +11,7 @@ import { XRInputSource, XRSession } from "three";
 import { Mathf } from "../engine/engine_math";
 import * as utils from "../engine/engine_three_utils"
 import { Interactable, UsageMarker } from "./Interactable";
-import { InstancingUtil } from "./Renderer";
+import { InstancingUtil } from "../engine/engine_instancing";
 import { Rigidbody } from "./RigidBody";
 import { getParam } from "../engine/engine_utils";
 
@@ -119,12 +119,22 @@ export class WebXRController extends Behaviour {
         return ctrl;
     }
 
+    // TODO: replace with component events
     public static addEventListener(evt: ControllerEvents, callback: (controller: WebXRController, args: any) => void) {
         const list = this.eventSubs[evt] ?? [];
         list.push(callback);
         this.eventSubs[evt] = list;
     }
 
+    // TODO: replace with component events
+    public static removeEventListener(evt: ControllerEvents, callback: (controller: WebXRController, args: any) => void) {
+        if(!callback) return;
+        const list = this.eventSubs[evt] ?? [];
+        const idx = list.indexOf(callback);
+        if (idx >= 0) list.splice(idx, 1);
+        this.eventSubs[evt] = list;
+    }
+
     private static eventSubs: { [key: string]: Function[] } = {};
 
     public webXR!: WebXR;