@needle-tools/engine 2.24.0-pre

package/src/engine-components/ui/EventSystem.ts

@@ -0,0 +1,523 @@
+import { RaycastOptions } from "../../engine/engine_physics";
+import { Behaviour, Component, GameObject } from "../Component";
+import { WebXR } from "../WebXR";
+import { ControllerEvents, WebXRController } from "../WebXRController";
+import * as ThreeMeshUI from 'three-mesh-ui'
+import { Context } from "../../engine/engine_setup";
+import { OrbitControls } from "../OrbitControls";
+import { PointerEventData } from "./PointerEvents";
+import { Raycaster } from "./Raycaster";
+import { InputEvents, KeyCode } from "../../engine/engine_input";
+import { Object3D } from "three";
+import { ICanvasGroup } from "./Interfaces";
+import { getParam } from "../../engine/engine_utils";
+import { off } from "process";
+
+const debug = getParam("debugeventsystem");
+
+export class EventSystem extends Behaviour {
+
+    static didSearchEventSystem: boolean = false;
+    static createIfNoneExists(context: Context) {
+        if (!this.didSearchEventSystem) {
+            this.didSearchEventSystem = true;
+            if (EventSystem.systems.length <= 0) {
+                EventSystem.systems.push(...GameObject.findObjectsOfType(EventSystem, context));
+            }
+        }
+        for (const sys of EventSystem.systems) {
+            if (sys.context === context) return; // exists
+        }
+        const go = new Object3D();
+        GameObject.addNewComponent(go, EventSystem);
+        context.scene.add(go);
+    }
+    static systems: EventSystem[] = [];
+
+    //@ts-ignore
+    static ensureUpdateMeshUI(instance, context: Context) {
+        MeshUIHelper.update(instance, context);
+    }
+    static markUIDirty(_context: Context) {
+        MeshUIHelper.markDirty();
+    }
+
+    static get instance(): EventSystem | null {
+        return this.systems[0];
+    }
+
+    private orbitControl: OrbitControls | null = null;
+    private orbitControlWasEnabled: boolean = false;
+    private raycaster: Raycaster[] = [];
+
+    constructor() {
+        super();
+        EventSystem.systems.push(this);
+    }
+
+    onDestroy(): void {
+        EventSystem.systems.splice(EventSystem.systems.indexOf(this), 1);
+    }
+
+    start() {
+        // const res = GameObject.findObjectsOfType(Raycaster, this.context);
+        // if (res)
+        //     this.raycaster = [...res];
+    }
+
+    register(rc: Raycaster) {
+        if (rc && this.raycaster && !this.raycaster.includes(rc))
+            this.raycaster?.push(rc);
+    }
+    unregister(rc: Raycaster) {
+        const i = this.raycaster?.indexOf(rc);
+        if (i !== undefined && i !== -1) {
+            this.raycaster?.splice(i, 1);
+        }
+    }
+
+    onEnable(): void {
+
+        WebXRController.addEventListener(ControllerEvents.SelectStart, (ctrl, args: { grab: THREE.Object3D | null }) => {
+            if (!args.grab) return;
+            MeshUIHelper.resetLastSelected();
+            const opts = new PointerEventData();
+            opts.inputSource = ctrl;
+            opts.isDown = ctrl.selectionDown;
+            opts.isUp = ctrl.selectionUp;
+            opts.isPressed = ctrl.selectionPressed;
+            opts.isClicked = false;
+            if (args.grab && !this.handleEvents(args.grab, opts)) {
+                args.grab = null;
+            }
+        });
+
+        const controllerRcOpts = new RaycastOptions();
+        WebXRController.addEventListener(ControllerEvents.SelectEnd, (ctrl, args: { grab: THREE.Object3D }) => {
+            if (!args.grab) return;
+            const opts = new PointerEventData();
+            opts.inputSource = ctrl;
+            opts.isDown = ctrl.selectionDown;
+            opts.isUp = ctrl.selectionUp;
+            opts.isPressed = ctrl.selectionPressed;
+            opts.isClicked = ctrl.selectionClick;
+            this.handleEvents(args.grab, opts);
+        });
+
+        WebXRController.addEventListener(ControllerEvents.Update, (_ctrl: WebXRController) => {
+            controllerRcOpts.ray = _ctrl.getRay();
+            const rc = this.performRaycast(controllerRcOpts);
+            if (!rc) return;
+            const opts = new PointerEventData();
+            opts.inputSource = _ctrl;
+            this.handleIntersections(rc, opts);
+        });
+
+        this.context.pre_update_callbacks.push(this.onBeforeUpdate.bind(this));
+        this.context.input.addEventListener(InputEvents.PointerDown, this.onPointerDown.bind(this));
+    }
+
+    onDisable(): void {
+        // TODO: unsubscribe from controller events
+    }
+
+    onPointerDown() {
+        this.onBeforeUpdate();
+    }
+
+    // doesnt work in dist
+    // onBeforeRender() {
+    //     MeshUIHelper.update(this.context);
+    // }
+
+    private lastPointerEvent: PointerEventData | null = null;
+    private objectsHoveredThisFrame: THREE.Object3D[] = [];
+    private objectsHoveredLastFrame: THREE.Object3D[] = [];
+    // todo: these should be listed by input source (e.g. per controller, mouse, touch)
+    private raisedPointerDownEvents: any[] = [];
+    private _didMove : boolean = false;
+
+    onBeforeUpdate() {
+        this.objectsHoveredThisFrame.length = 0;
+        this.resetMeshUIStates();
+
+        if (WebXR.IsInWebXR) return;
+        // console.log(this.context.input.isKeyPressed(KeyCode.ALT));
+        if (this.context.input.isKeyPressed(KeyCode.ALT)) {
+            // console.log("alt pressed");
+            return;
+        }
+        if(!this._didMove){
+            const pos = this.context.input.getPointerPositionRC(0);
+            if(pos && pos.x === 0 && pos.y === 0) return;
+            this._didMove = true;
+        }
+
+        const hits = this.performRaycast(null);
+        const args: PointerEventData = new PointerEventData(this.context.input.getPointerEvent(0));
+        args.inputSource = this.context.input;
+        args.isClicked = this.context.input.mouseClick;
+        args.isDown = this.context.input.mouseDown;
+        args.isUp = this.context.input.mouseUp;
+        args.isPressed = this.context.input.mousePressed;
+        this.lastPointerEvent = args;
+        if (!hits) return;
+
+        // handle orbit control before handling input events
+        // this is just so button events that modify OrbitControls.enabled work
+        let orbitControlComponent: OrbitControls | null = null;
+        let previousOrbitControlState : boolean | null = null;
+        if (this.context.input.mouseDown && this.currentActiveMeshUIComponents.length > 0 && this.context.mainCameraComponent) {
+            orbitControlComponent = GameObject.getComponent(this.context.mainCameraComponent.gameObject, OrbitControls) ?? null;
+            if(orbitControlComponent){
+                previousOrbitControlState = orbitControlComponent.enabled;
+                orbitControlComponent.enabled = false;
+            }
+        }
+
+        // if (args.isClicked)
+        //     console.log(this.guid, ...hits);
+        this.handleIntersections(hits, args);
+
+        if (orbitControlComponent) {
+            this.orbitControl = orbitControlComponent;
+            if (this.orbitControl?.enabled) {
+                this.orbitControlWasEnabled = this.orbitControl.enabled;
+                this.orbitControl.enabled = false;
+            }
+            else if (this.orbitControl && !this.context.input.mousePressed) {
+                this.orbitControl.enabled = this.orbitControlWasEnabled;
+                this.orbitControl = null;
+            }
+        }
+
+    }
+
+    private _tempComponentsArray: Behaviour[] = [];
+    onBeforeRender() {
+        if (this.lastPointerEvent)
+            this.lastPointerEvent.used = false;
+        else return;
+
+        if (this.lastPointerEvent.isUp) {
+            for (const obj of this.raisedPointerDownEvents) {
+                if (obj.onPointerUp) {
+                    obj.onPointerUp(this.lastPointerEvent);
+                }
+            }
+            this.raisedPointerDownEvents.length = 0;
+        }
+
+        for (const obj of this.objectsHoveredLastFrame) {
+            // if the object is not hovered anymore we want to call the exit events
+            if (this.objectsHoveredThisFrame.indexOf(obj) < 0) {
+                // console.log("LAST", ...this.objectsHoveredLastFrame);
+                // console.log("THIS", ...this.objectsHoveredThisFrame);
+                this._tempComponentsArray.length = 0;
+                const behaviours = GameObject.getComponentsInParent(obj, Behaviour, this._tempComponentsArray);
+                this.lastPointerEvent.object = obj;
+                for (let i = 0; i < behaviours.length; i++) {
+                    const beh = behaviours[i];
+                    if (!beh.gameObject || beh.destroyed) continue;
+                    const inst: any = beh;
+                    if (inst.onPointerExit) {
+                        inst.onPointerExit(this.lastPointerEvent);
+                    }
+                }
+            }
+        }
+        // swap arrays
+        const arr = this.objectsHoveredLastFrame;
+        this.objectsHoveredLastFrame = this.objectsHoveredThisFrame;
+        this.objectsHoveredThisFrame = arr;
+    }
+
+    private _sortedHits: THREE.Intersection[] = [];
+
+    private performRaycast(opts: RaycastOptions | null): THREE.Intersection[] | null {
+        if (!this.raycaster) return null;
+        this._sortedHits.length = 0;
+        for (const rc of this.raycaster) {
+            if (!rc.activeAndEnabled) continue;
+            const res = rc.performRaycast(opts);
+            if (res && res.length > 0)
+                this._sortedHits.push(...res);
+        }
+        this._sortedHits.sort((a, b) => {
+            return a.distance - b.distance;
+        });
+        return this._sortedHits;
+    }
+
+    private handleIntersections(hits: THREE.Intersection[], args: PointerEventData): boolean {
+        if (!hits || hits.length <= 0) return false;
+        // if (pressed)
+        //     console.log(this.context.alias, ...hits);
+        hits = this.sortCandidates(hits);
+        for (const hit of hits) {
+            const { object } = hit;
+            if (this.handleEvents(object, args)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private _sortingBuffer: THREE.Intersection[] = [];
+    private _noDepthTestingResults: THREE.Intersection[] = [];
+
+    private sortCandidates(hits: THREE.Intersection[]): THREE.Intersection[] {
+        // iterate over all hits and filter for nodepth objects and normal hit objects
+        // the no-depth objects will be handled first starting from the closest
+        // assuming the hits array is sorted by distance (closest > furthest)
+        this._sortingBuffer.length = 0;
+        this._noDepthTestingResults.length = 0;
+        for (let i = 0; i < hits.length; i++) {
+            const hit = hits[i];
+            const object = hit.object as THREE.Mesh;   
+            if(object.material){
+                if(object.material["depthTest"] === false){
+                    this._noDepthTestingResults.push(hit);
+                    continue;
+                }
+            }
+            this._sortingBuffer.push(hit);
+        }
+        for(const obj of this._sortingBuffer){
+            this._noDepthTestingResults.push(obj);
+        }
+        return this._noDepthTestingResults;
+    }
+
+    private handleEventsArray: Array<Behaviour> = [];
+    private handleEvents(object: THREE.Object3D, args: PointerEventData): boolean {
+
+        if (!this.testIsVisible(object)) {
+            if (args.isClicked && debug)
+                console.log("not allowed", object);        
+            return false;
+        }
+
+        args.object = object;
+        this.lastPointerEvent = args;
+
+        const parent = object.parent as any;
+        let isShadow = false;
+        const clicked = args.isClicked ?? false;
+
+        let canvasGroup: ICanvasGroup | null = null;
+
+        // handle potential shadow dom built from three mesh ui
+        if (parent && parent.isUI) {
+            const pressedOrClicked = (args.isPressed || args.isClicked) ?? false;
+            if (parent.shadowComponentOwner) {
+                const actualGo = parent.shadowComponentOwner.gameObject;
+                if (actualGo) {
+                    canvasGroup = this.tryFindCanvasGroup(actualGo);
+                    if (canvasGroup?.blocksRaycasts === false) return false;
+                    if (canvasGroup?.interactable === false) {
+                        return true;
+                    }
+                    const handled = this.handleMeshUIIntersection(object, pressedOrClicked);
+                    if (!clicked && handled) {
+                        // return true;
+                    }
+                    object = actualGo;
+                    isShadow = true;
+                }
+            }
+
+            if (!isShadow) {
+                const obj = this.handleMeshUiObjectWithoutShadowDom(parent, pressedOrClicked);
+                if (obj) return true;
+            }
+        }
+
+        this.objectsHoveredThisFrame.push(object);
+
+        if (canvasGroup === null || canvasGroup.interactable) {
+            const isHovered = this.objectsHoveredLastFrame.indexOf(object) >= 0;
+            this.handleEventsArray.length = 0;
+            const behaviours = GameObject.getComponentsInParent(object, Behaviour, this.handleEventsArray);
+            // console.log(behaviours);
+            for (let i = 0; i < behaviours.length; i++) {
+                if (args.used) return true;
+                if (behaviours[i].destroyed) continue;
+                const comp = behaviours[i] as any;
+
+                if (comp.interactable === false) continue;
+
+                if (comp.onPointerEnter) {
+                    if (!isHovered) {
+                        comp.onPointerEnter(args);
+                    }
+                }
+
+                if (args.isDown) {
+                    if (comp.onPointerDown && !this.raisedPointerDownEvents.includes(comp)) {
+                        comp.onPointerDown(args);
+                        // need to save this to also send pointer up event
+                        this.raisedPointerDownEvents.push(comp);
+                    }
+                }
+
+                if (args.isUp) {
+                    if (comp.onPointerUp) {
+                        comp.onPointerUp(args);
+                    }
+                }
+
+                if (args.isClicked) {
+                    if (comp.onPointerClick) {
+                        comp.onPointerClick(args);
+                    }
+                }
+            }
+        }
+
+        return true;
+    }
+
+    private handleMeshUiObjectWithoutShadowDom(obj: any, pressed: boolean) {
+        if (!obj || !obj.isUI) return true;
+        const hit = this.handleMeshUIIntersection(obj, pressed);
+
+        return hit;
+    }
+
+    private currentActiveMeshUIComponents: ThreeMeshUI.Block[] = [];
+
+    private handleMeshUIIntersection(meshUiObject: THREE.Object3D, pressed: boolean): boolean {
+        const res = MeshUIHelper.updateState(meshUiObject, pressed);
+        if (res) {
+            this.currentActiveMeshUIComponents.push(res);
+        }
+        return res !== null;
+    }
+
+    private resetMeshUIStates() {
+
+        if (this.context.input.getPointerPressedCount() > 0) {
+            MeshUIHelper.resetLastSelected();
+        }
+
+        if (!this.currentActiveMeshUIComponents || this.currentActiveMeshUIComponents.length <= 0) return;
+        for (let i = 0; i < this.currentActiveMeshUIComponents.length; i++) {
+            const comp = this.currentActiveMeshUIComponents[i];
+            MeshUIHelper.resetState(comp);
+        }
+        this.currentActiveMeshUIComponents.length = 0;
+    }
+
+    private testIsVisible(obj: THREE.Object3D | null): boolean {
+        if (!obj) return true;
+        if (!GameObject.isActiveSelf(obj)) return false;
+        return this.testIsVisible(obj.parent);
+    }
+
+    private tryFindCanvasGroup(obj: THREE.Object3D | null): ICanvasGroup | null {
+        if (!obj) return null;
+        // test for canvas groups
+        const res = GameObject.foreachComponent(obj, c => {
+            const gr = c as unknown as ICanvasGroup;
+            if (gr.blocksRaycasts !== undefined && gr.interactable !== undefined) return gr;
+            return undefined;
+        }, false);
+        if (res !== undefined) return res;
+        return this.tryFindCanvasGroup(obj.parent);
+    }
+}
+
+
+class MeshUIHelper {
+
+    private static lastSelected: THREE.Object3D | null = null;
+    private static lastUpdateFrame: { context: Context, frame: number }[] = [];
+    private static needsUpdate: boolean = false;
+
+    static markDirty() {
+        this.needsUpdate = true;
+    }
+
+    static update(threeMeshUI: any, context: Context) {
+        for (const lu of this.lastUpdateFrame) {
+            if (lu.context === context) {
+                if (context.time.frameCount === lu.frame) return;
+                lu.frame = context.time.frameCount;
+                if (this.needsUpdate || context.time.frameCount < 30) {
+                    this.needsUpdate = false;
+                    threeMeshUI.update();
+                }
+                return;
+            }
+        }
+        this.lastUpdateFrame = [{ context, frame: context.time.frameCount }];
+        threeMeshUI.update();
+    }
+
+    static updateState(intersect: THREE.Object3D, selectState: boolean): ThreeMeshUI.Block | null {
+        let foundBlock: ThreeMeshUI.Block | null = null;
+
+        if (intersect) {
+            foundBlock = this.findBlockInParent(intersect);
+            // console.log(intersect, "-- found block:", foundBlock)
+            if (foundBlock && foundBlock !== this.lastSelected) {
+                const interactable = foundBlock["interactable"];
+                if (interactable === false) return null;
+                if (selectState) {
+                    this.lastSelected = foundBlock;
+                    //@ts-ignore
+                    if (foundBlock.states["pressed"])
+                        //@ts-ignore
+                        foundBlock.setState("pressed");
+
+                } else {
+                    //@ts-ignore
+                    if (foundBlock.states["hovered"])
+                        //@ts-ignore
+                        foundBlock.setState("hovered");
+                };
+                this.needsUpdate = true;
+            }
+        }
+
+        // Update non-targeted buttons state
+        // MeshUIBaseComponent.objectsWithState.forEach((obj) => {
+        //     if ((!intersect || obj !== foundBlock) && obj.isUI) {
+        //         obj.setState('idle');
+        //     };
+        // });
+
+        return foundBlock;
+    }
+
+    static resetLastSelected() {
+        const last = this.lastSelected;
+        if (!last) return;
+        this.lastSelected = null;
+        this.resetState(last);
+    }
+
+    static resetState(obj: any) {
+        if (!obj) return;
+        const interactable = obj["interactable"];
+        if (interactable === false) {
+            if (obj.states["disabled"])
+                obj.setState("disabled");
+        }
+        else if (obj === this.lastSelected && obj.states["selected"]) {
+            obj.setState("selected");
+        }
+        else obj.setState('normal');
+        this.needsUpdate = true;
+    }
+
+    static findBlockInParent(elem: any): ThreeMeshUI.Block | null {
+        if (!elem) return null;
+        if (elem.isBlock) {
+            if (Object.keys(elem.states).length > 0)
+                return elem;
+        }
+        return this.findBlockInParent(elem.parent);
+    }
+}

package/src/engine-components/ui/Graphic.ts

@@ -0,0 +1,157 @@
+import { IGraphic } from './Interfaces';
+import * as ThreeMeshUI from 'three-mesh-ui'
+import { RGBAColor } from "../js-extensions/RGBAColor"
+import { BaseUIComponent } from "./BaseUIComponent";
+import { serializeable } from '../../engine/engine_serialization_decorator';
+import { Color, Texture } from 'three';
+import { RectTransform } from './RectTransform';
+import { onChange, scheduleAction } from "./Utils"
+
+
+export class Graphic extends BaseUIComponent implements IGraphic {
+
+    @serializeable(RGBAColor)
+    get color(): RGBAColor {
+        if (!this._color) this._color = new RGBAColor(1, 1, 1, 1);
+        return this._color;
+    }
+    set color(col: RGBAColor) {
+        const changed = !this._color || this._color.r !== col.r || this._color.g !== col.g || this._color.b !== col.b || this._color.alpha !== col.alpha;
+        if (!changed) return;
+        if (!this._color) {
+            this._color = new RGBAColor(1, 1, 1, 1);
+        }
+        this._color.copy(col);
+    }
+    protected onColorChanged() {
+        const newcolor = this.color;
+        this.setOptions({ backgroundColor: newcolor, backgroundOpacity: newcolor.alpha, borderOpacity: newcolor.alpha });
+    }
+
+    // used via animations
+    private get m_Color() {
+        return this._color;
+    }
+
+    @serializeable()
+    raycastTarget: boolean = true;
+
+    protected uiObject: ThreeMeshUI.Block | null = null;
+    private _color: RGBAColor = null!;
+
+
+    private _rect: RectTransform | null = null;
+    protected get rectTransform(): RectTransform {
+        if (!this._rect) {
+            this._rect = this.gameObject.getComponent(RectTransform);
+        }
+        return this._rect!;
+    }
+
+    setState(state: string) {
+        this.makePanel();
+        if (this.uiObject) {
+            //@ts-ignore
+            this.uiObject.setState(state);
+        }
+    }
+
+    setupState(state: object) {
+        this.makePanel();
+        if (this.uiObject) {
+            //@ts-ignore
+            this.uiObject.setupState(state);
+        }
+    }
+
+    setOptions(opts: object) {
+        this.makePanel();
+        if (this.uiObject) {
+            //@ts-ignore
+            this.uiObject.set(opts);
+            if (opts["backgroundColor"] !== undefined || opts["backgroundOpacity"] !== undefined)
+                this.uiObject["updateBackgroundMaterial"]?.call(this.uiObject);
+        }
+    }
+
+    awake() {
+        super.awake();
+        this.makePanel();
+
+        // when _color is written to
+        onChange(this, "_color", () => scheduleAction(this, this.onColorChanged));
+    }
+
+    onEnable(): void {
+        super.onEnable();
+        if (this.uiObject) {
+            this.rectTransform.shadowComponent?.add(this.uiObject);
+            this.addShadowComponent(this.uiObject, this.rectTransform);
+        }
+
+    }
+
+    onDisable(): void {
+        super.onDisable();
+        if (this.uiObject)
+            this.removeShadowComponent();
+    }
+
+    private _currentlyCreatingPanel: boolean = false;
+    protected makePanel() {
+        if (this.uiObject) return;
+        if (this._currentlyCreatingPanel) return;
+        this._currentlyCreatingPanel = true;
+
+        const opts = {
+            backgroundColor: this.color,
+            backgroundOpacity: this.color.alpha,
+            offset: 1, // without a tiny offset we get z fighting
+        };
+        this.onBeforeCreate(opts);
+        this.onCreate(opts);
+        this.controlsChildLayout = false;
+        this._currentlyCreatingPanel = false;
+        this.onAfterCreated();
+    }
+
+    protected onBeforeCreate(_opts: any) { }
+
+    protected onCreate(opts: any) {
+        this.uiObject = this.rectTransform.createNewBlock(opts);
+        this.uiObject.name = this.name;
+    }
+    protected onAfterCreated() { }
+
+    protected async setTexture(tex: Texture | null | undefined) {
+        if (!tex) return;
+        this.setOptions({ backgroundOpacity: 0 });
+        if (tex) {
+            this.setOptions({ backgroundTexture: tex, borderRadius: 0, backgroundOpacity: this.color.alpha, backgroundSize: "stretch" });
+        }
+    }
+
+    protected onAfterAddedToScene(): void {
+        if (this.shadowComponent) {
+            //@ts-ignore
+            this.shadowComponent.offset = this.shadowComponent.position.z;
+
+            // console.log(this.shadowComponent);
+            // setTimeout(()=>{
+            //     this.shadowComponent?.traverse(c => {
+            //         console.log(c);
+            //         if(c.material) c.material.depthTest = false;
+            //     });
+            // },1000);
+        }
+    }
+}
+
+export class MaskableGraphic extends Graphic {
+
+    protected onAfterCreated() {
+        // flip image
+        if (this.uiObject)
+            this.uiObject.scale.y *= -1;
+    }
+}