@needle-tools/engine 2.24.0-pre

package/src/engine-components/TestRunner.ts

@@ -0,0 +1,116 @@
+import { Behaviour } from "./Component";
+import * as tests from "../engine/tests/test_utils";
+import { createTransformModel, SyncedTransform, SyncedTransformIdentifier } from "./SyncedTransform";
+import * as flatbuffers from 'flatbuffers';
+import { SyncedTransformModel } from "../engine-schemes/synced-transform-model";
+import * as schemes from "../engine-schemes/schemes";
+import { IModel, NetworkConnection } from "../engine/engine_networking";
+import { Rigidbody } from "./Rigidbody";
+import { Vector3 } from "three";
+
+export class TestRunner extends Behaviour {
+    awake(): void {
+        tests.detect_run_tests();
+    }
+}
+
+export class TestSimulateUserData extends Behaviour {
+
+    transformsPerFrame: number = 10;
+    interval: number = 0;
+    useFlatbuffers: boolean = true;
+
+    awake(): void {
+        if (this.useFlatbuffers) {
+            this.context.connection.beginListenBinrary(SyncedTransformIdentifier, (_mod: SyncedTransformModel) => {
+                // console.log("Received transform");
+                // const sc = SyncedTransformModel.getRootAsSyncedTransformModel(bin);
+                // console.log(mod.guid());
+                // console.log("Received transform", sc, sc.transform()?.position()?.x(), sc.fast(), bin.getBufferIdentifier());
+            });
+        }
+        else {
+            this.models = [];
+            for (let i = 0; i < this.transformsPerFrame; i++) {
+                this.models.push(new TransformModel(this.context.connection.connectionId + "_simulatedTransform_" + i, this));
+            }
+        }
+    }
+
+    private builder: flatbuffers.Builder | null = null;
+    private models: TransformModel[] | null = null;
+
+    update() {
+        if (!this.context.connection.isConnected) return;
+
+        if (this.useFlatbuffers) {
+            if (!this.context.connection.connectionId || this.context.time.frameCount % this.interval !== 0) return;
+            if (this.builder === null)
+                this.builder = new flatbuffers.Builder(1024);
+            const builder = this.builder;
+            for (let i = 0; i < this.transformsPerFrame; i++) {
+                builder.clear();
+                const buf = createTransformModel(this.context.connection.connectionId!, this);
+                this.context.connection.sendBinary(buf);
+            }
+        }
+        else {
+            if (this.models) {
+                for (let i = 0; i < this.models.length; i++) {
+                    const mod = this.models[i];
+                    mod.dontSave = true;
+                    mod.update(this, null);
+                    this.context.connection.send("TestSimulateUserData-" + i, mod);
+                }
+            }
+        }
+    }
+
+}
+
+
+// use flatbuffer SyncedTransformModel
+class TransformModel implements IModel {
+    guid: string;
+    fast: boolean = false;
+    position: { x: number, y: number, z: number };
+    rotation: { x: number, y: number, z: number, w: number };
+    // scale : { x : number, y : number, z : number } | undefined = undefined;
+    velocity: { x: number, y: number, z: number } | undefined = undefined;
+    dontSave?: boolean | undefined;
+
+    isValid(): boolean {
+        return this.fast !== undefined || this.position !== undefined || this.rotation !== undefined || this.velocity !== undefined;
+    }
+
+    constructor(guid: string, obj: Behaviour) {
+        this.guid = guid;
+        this.position = { x: 0, y: 0, z: 0 };
+        this.rotation = { x: 0, y: 0, z: 0, w: 0 };
+        this.update(obj, null)
+    }
+    public static temp: THREE.Vector3 = new Vector3();
+
+    update(beh: Behaviour, rb: Rigidbody | undefined | null) {
+        const world = beh.worldPosition;// go.getWorldPosition(TransformModel.temp);
+        this.position.x = world.x;
+        this.position.y = world.y;
+        this.position.z = world.z;
+
+        const rot = beh.worldQuaternion;
+        this.rotation.x = rot.x;
+        this.rotation.y = rot.y;
+        this.rotation.z = rot.z;
+        this.rotation.w = rot.w;
+
+        this.fast = false;
+
+        if (rb) {
+            const vel = rb.getVelocity();
+            if (this.velocity === undefined) this.velocity = { x: 0, y: 0, z: 0 };
+            this.velocity.x = vel.x;
+            this.velocity.y = vel.y;
+            this.velocity.z = vel.z;
+        }
+    }
+}

package/src/engine-components/TransformGizmo.ts

@@ -0,0 +1,185 @@
+import { Behaviour, GameObject } from "./Component";
+import { SyncedTransform } from "./SyncedTransform";
+import { serializeable } from "../engine/engine_serialization_decorator";
+import * as params from "../engine/engine_default_parameters";
+import { Mesh, MathUtils, EventListener } from "three";
+import { TransformControls } from "three/examples/jsm/controls/TransformControls";
+import { OrbitControls } from "three/examples/jsm/controls/OrbitControls";
+
+export class TransformGizmo extends Behaviour {
+
+    @serializeable()
+    public isGizmo: boolean = true;
+
+    private control?: TransformControls;
+    private orbit?: OrbitControls;
+
+    awake() {
+        if (this.isGizmo && !params.showGizmos) return;
+        if (!this.context.mainCamera) return;
+        this.control = new TransformControls(this.context.mainCamera, this.context.renderer.domElement);
+        this.control.visible = true;
+        this.control.enabled = true;
+        this.control.getRaycaster().layers.set(2);
+
+        this.control.size = 0.6;
+        this.control.traverse(x => {
+            const mesh = x as Mesh;
+            mesh.layers.set(2);
+            if (mesh) {
+                const gizmoMat = mesh.material as THREE.MeshBasicMaterial;
+                if (gizmoMat) {
+                    gizmoMat.opacity = 0.3;
+                }
+            }
+        });
+    }
+
+    start() {
+        if (this.context.mainCamera) {
+            const orbit = GameObject.getComponentInParent(this.context.mainCamera, OrbitControls) ?? undefined;
+            this.orbit = orbit;
+        }
+    }
+
+    private changeEventListener?: any;
+    private windowKeyDownListener?: any;
+    private windowKeyUpListener?: any;
+
+    onEnable() {
+        if (this.control) {
+            this.context.scene.add(this.control);
+            this.control.attach(this.gameObject);
+        }
+        this.changeEventListener = this.onControlChangedEvent.bind(this);
+        this.control?.addEventListener('dragging-changed', this.changeEventListener);
+        this.attachWindowEvents();
+    }
+
+    onDisable() {
+        this.control?.removeFromParent();
+        if (this.changeEventListener)
+            this.control?.removeEventListener('dragging-changed', this.changeEventListener);
+    }
+
+    private onControlChangedEvent(event) {
+        const orbit = this.orbit;
+        if (orbit) orbit.enabled = !event.value;
+        if (event.value) {
+            // request ownership on drag start
+            const sync = GameObject.getComponentInParent(this.gameObject, SyncedTransform);
+            if (sync) {
+                sync.requestOwnership();
+            }
+        }
+    }
+
+    private attachWindowEvents() {
+        const control = this.control;
+        if (!control) return;
+
+        if (!this.windowKeyDownListener) {
+            this.windowKeyDownListener = (event) => {
+                switch (event.keyCode) {
+
+                    case 81: // Q
+                        control.setSpace(control.space === 'local' ? 'world' : 'local');
+                        break;
+
+                    case 16: // Shift
+                        control.setTranslationSnap(100);
+                        control.setRotationSnap(MathUtils.degToRad(15));
+                        control.setScaleSnap(0.25);
+                        break;
+
+                    case 87: // W
+                        control.setMode('translate');
+                        break;
+
+                    case 69: // E
+                        control.setMode('rotate');
+                        break;
+
+                    case 82: // R
+                        control.setMode('scale');
+                        break;
+
+                    /*
+                    case 67: // C
+                        const position = currentCamera.position.clone();
+    
+                        currentCamera = currentCamera.isPerspectiveCamera ? cameraOrtho : cameraPersp;
+                        currentCamera.position.copy( position );
+    
+                        orbit.object = currentCamera;
+                        control.camera = currentCamera;
+    
+                        currentCamera.lookAt( orbit.target.x, orbit.target.y, orbit.target.z );
+                        onWindowResize();
+                        break;
+    
+                    case 86: // V
+                        const randomFoV = Math.random() + 0.1;
+                        const randomZoom = Math.random() + 0.1;
+    
+                        cameraPersp.fov = randomFoV * 160;
+                        cameraOrtho.bottom = - randomFoV * 500;
+                        cameraOrtho.top = randomFoV * 500;
+    
+                        cameraPersp.zoom = randomZoom * 5;
+                        cameraOrtho.zoom = randomZoom * 5;
+                        onWindowResize();
+                        break;
+                    */
+                    case 187:
+                    case 107: // +, =, num+
+                        control.setSize(control.size + 0.1);
+                        break;
+
+                    case 189:
+                    case 109: // -, _, num-
+                        control.setSize(Math.max(control.size - 0.1, 0.1));
+                        break;
+
+                    case 88: // X
+                        control.showX = !control.showX;
+                        break;
+
+                    case 89: // Y
+                        control.showY = !control.showY;
+                        break;
+
+                    case 90: // Z
+                        control.showZ = !control.showZ;
+                        break;
+
+                    case 32: // Spacebar
+                        control.enabled = !control.enabled;
+                        break;
+
+                }
+
+            };
+        }
+
+        if (!this.windowKeyUpListener) {
+            this.windowKeyUpListener = (event) => {
+
+                switch (event.keyCode) {
+
+                    case 16: // Shift
+                        control.setTranslationSnap(null);
+                        control.setRotationSnap(null);
+                        control.setScaleSnap(null);
+                        break;
+
+                }
+
+            };
+        }
+
+        
+        window.addEventListener('keydown', this.windowKeyDownListener);
+        window.addEventListener('keyup', this.windowKeyUpListener);
+    }
+}

package/src/engine-components/VideoPlayer.ts

@@ -0,0 +1,308 @@
+import { Behaviour } from "./Component";
+import * as THREE from "three";
+import { Material } from "material/Material";
+import { serializeable } from "../engine/engine_serialization_decorator";
+import { Object3D } from "three";
+import { awaitInput } from "../engine/engine_input_utils";
+import { getParam } from "../engine/engine_utils";
+
+const debug = getParam("debugvideo");
+
+export enum VideoSource {
+    /// <summary>
+    ///   <para>Use the current clip as the video content source.</para>
+    /// </summary>
+    VideoClip,
+    /// <summary>
+    ///   <para>Use the current URL as the video content source.</para>
+    /// </summary>
+    Url,
+}
+
+export enum VideoAudioOutputMode {
+    /// <summary>
+    ///   <para>Disable the embedded audio.</para>
+    /// </summary>
+    None,
+    /// <summary>
+    ///   <para>Send the embedded audio into a specified AudioSource.</para>
+    /// </summary>
+    AudioSource,
+    /// <summary>
+    ///   <para>Send the embedded audio direct to the platform's audio hardware.</para>
+    /// </summary>
+    Direct,
+    /// <summary>
+    ///   <para>Send the embedded audio to the associated AudioSampleProvider.</para>
+    /// </summary>
+    APIOnly,
+}
+
+export class VideoPlayer extends Behaviour {
+
+    @serializeable(Object3D)
+    renderer: THREE.Object3D | null = null;
+    @serializeable()
+    playOnAwake: boolean = true;
+    @serializeable()
+    playOnEnable?: boolean;
+
+    @serializeable()
+    targetMaterialProperty?: string;
+
+    @serializeable()
+    private time: number = 0;
+
+    private _playbackSpeed: number = 1;
+    @serializeable()
+    get playbackSpeed(): number {
+        return this.videoElement?.playbackRate ?? this._playbackSpeed;
+    }
+    set playbackSpeed(val: number) {
+        this._playbackSpeed = val;
+        if (this.videoElement)
+            this.videoElement.playbackRate = val;
+    }
+
+    private _isLooping: boolean = false;
+    get isLooping(): boolean {
+        return this.videoElement?.loop ?? this._isLooping;
+    }
+    @serializeable()
+    set isLooping(val: boolean) {
+        this._isLooping = val;
+        if (this.videoElement)
+            this.videoElement.loop = val;
+    }
+
+    get currentTime(): number {
+        return this.videoElement?.currentTime ?? this.time;
+    }
+    set currentTime(val: number) {
+        if (this.videoElement) {
+            this.videoElement.currentTime = val;
+        }
+        else this.time = val;
+    }
+
+    get isPlaying(): boolean {
+        const video = this.videoElement;
+        if (video) {
+            return video.currentTime > 0 && !video.paused && !video.ended
+                && video.readyState > video.HAVE_CURRENT_DATA;
+        }
+        return false;
+    }
+
+    get crossOrigin(): string | null {
+        return this.videoElement?.crossOrigin ?? this._crossOrigin;
+    }
+    set crossOrigin(val: string | null) {
+        this._crossOrigin = val;
+        if (this.videoElement) {
+            if (val !== null) this.videoElement.setAttribute("crossorigin", val);
+            else this.videoElement.removeAttribute("crossorigin");
+        }
+    }
+
+    private _crossOrigin: string | null = "anonymous";
+
+    private audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.AudioSource;
+
+    private source!: VideoSource;
+    private clip?: string | MediaStream | null = null;
+    private url?: string | null = null;
+
+    private videoElement: HTMLVideoElement | null = null;
+    private videoTexture: THREE.VideoTexture | null = null;
+    private videoMaterial: Material | null = null;
+
+    private _isPlaying: boolean = false;
+    private wasPlaying: boolean = false;
+
+    setClipURL(url: string) {
+        if (this.url === url) return;
+        // console.log("SET URL", url);
+        this.url = url;
+        this.source = VideoSource.Url;
+        if (debug) console.log("set url", url);
+        if (this.videoElement) {
+            this.videoElement.src = url;
+            if (this._isPlaying)
+                this.videoElement.play();
+        }
+    }
+
+    awake(): void {
+        this.create(this.playOnAwake);
+
+        window.addEventListener('visibilitychange', _evt => {
+            switch (document.visibilityState) {
+                case "hidden":
+                    this.wasPlaying = this._isPlaying;
+                    this.pause();
+                    break;
+                case "visible":
+                    if (this.wasPlaying) this.play();
+                    break;
+            }
+        });
+    }
+
+    onEnable(): void {
+        if (this.playOnEnable === true) {
+            this.handleBeginPlaying(true);
+        }
+    }
+
+    onDisable(): void {
+        this.pause();
+    }
+
+    onDestroy(): void {
+        if (this.videoElement) {
+            this.videoElement.parentElement?.removeChild(this.videoElement);
+            this.videoElement = null;
+        }
+        if (this.videoTexture) {
+            this.videoTexture.dispose();
+            this.videoTexture = null;
+        }
+    }
+
+    private _receivedInput: boolean = false;
+
+    constructor() {
+        super();
+        awaitInput(() => {
+            this._receivedInput = true;
+            this.updateVideoElementSettings();
+        })
+    }
+
+    play() {
+        if (!this.videoElement) return;
+        if (this._isPlaying && !this.videoElement?.ended && !this.videoElement?.paused) return;
+        this._isPlaying = true;
+        if (!this._receivedInput) this.videoElement.muted = true;
+        this.updateVideoElementSettings();
+        this.videoElement?.play().catch(err => {
+            console.warn(err);
+        });
+        if (debug) console.log("play", this.videoElement);
+    }
+
+    stop() {
+        this._isPlaying = false;
+        if (!this.videoElement) return;
+        this.videoElement.currentTime = 0;
+        this.videoElement.pause();
+    }
+
+    pause(): void {
+        this._isPlaying = false;
+        this.videoElement?.pause();
+    }
+
+
+    create(playAutomatically: boolean) {
+        let src;
+        switch (this.source) {
+            case VideoSource.VideoClip:
+                src = this.clip;
+                break;
+            case VideoSource.Url:
+                src = this.url;
+                break;
+        }
+
+        if (!src) return;
+
+        // console.log(src, this);
+
+        if (!this.videoElement) {
+            this.videoElement = this.createVideoElement();
+            this.context.domElement?.prepend(this.videoElement);
+            // hide it because otherwise it would overlay the website with default css
+            this.updateVideoElementStyles();
+        }
+        if (typeof src === "string")
+            this.videoElement.src = src;
+        else
+            this.videoElement.srcObject = src;
+
+        if (!this.videoTexture)
+            this.videoTexture = new THREE.VideoTexture(this.videoElement);
+        this.videoTexture.flipY = false;
+        this.videoTexture.encoding = THREE.sRGBEncoding;
+        this.handleBeginPlaying(playAutomatically);
+    }
+
+    private createVideoElement(): HTMLVideoElement {
+        const video = document.createElement("video") as HTMLVideoElement;
+        if (this._crossOrigin)
+            video.setAttribute("crossorigin", this._crossOrigin);
+        if (debug) console.log("create video elment", video);
+        return video;
+    }
+
+    private handleBeginPlaying(playAutomatically: boolean) {
+        if (!this.enabled) return;
+        if (!this.videoElement) return;
+
+        const mat = this.gameObject["material"];
+        if (mat) {
+            if (mat !== this.videoMaterial) {
+                this.videoMaterial = mat.clone();
+                this.gameObject["material"] = this.videoMaterial;
+            }
+
+            if (!this.targetMaterialProperty) {
+                (this.videoMaterial as any).map = this.videoTexture;
+            }
+            else {
+                switch (this.targetMaterialProperty) {
+                    default:
+                        (this.videoMaterial as any).map = this.videoTexture;
+                        break;
+                    // doesnt render:
+                    // case "emissiveTexture":
+                    //     console.log(this.videoMaterial);
+                    //     // (this.videoMaterial as any).map = this.videoTexture;
+                    //     (this.videoMaterial as any).emissive?.set(1,1,1);// = this.videoTexture;
+                    //     (this.videoMaterial as any).emissiveMap = this.videoTexture;
+                    //     break;
+                }
+            }
+        }
+        else {
+            console.warn("Can not play video, no material found, this might be a multimaterial case which is not supported yet");
+            return;
+        }
+        this.updateVideoElementSettings();
+        this.updateVideoElementStyles();
+        if (playAutomatically)
+            this.play();
+    }
+
+    private updateVideoElementSettings() {
+        if (!this.videoElement) return;
+        this.videoElement.loop = this._isLooping;
+        this.videoElement.currentTime = this.currentTime;
+        this.videoElement.playbackRate = this._playbackSpeed;
+        // dont open in fullscreen on ios
+        this.videoElement.playsInline = true;
+        this.videoElement.muted = !this._receivedInput && this.audioOutputMode !== VideoAudioOutputMode.None;
+        if (this.playOnAwake || this.playOnEnable)
+            this.videoElement.autoplay = true;
+    }
+
+    private updateVideoElementStyles() {
+        if (!this.videoElement) return;
+        // set style here so preview frame is rendered
+        // set display and selectable because otherwise is interfers with input/focus e.g. breaks orbit control
+        this.videoElement.style.userSelect = "none";
+        this.videoElement.style.visibility = "hidden";
+        this.videoElement.style.display = "none";
+    }
+}