@needle-tools/engine 3.3.0-alpha → 3.5.0-alpha

package/src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts

@@ -0,0 +1,459 @@
+import { Object3D } from "three";
+import { USDDocument, USDObject, USDWriter, makeNameSafeForUSD } from "../../ThreeUSDZExporter";
+
+import { BehaviorExtension } from "./Behaviour";
+
+// TODO: rename to usdz element
+export interface IBehaviorElement {
+    id: string;
+    writeTo(document: USDDocument, writer: USDWriter);
+}
+
+export class BehaviorModel {
+
+    static global_id: number = 0;
+    id: string;
+    trigger: IBehaviorElement | IBehaviorElement[];
+    action: IBehaviorElement;
+    exclusive: boolean = false;
+
+    makeExclusive(exclusive: boolean): BehaviorModel {
+        this.exclusive = exclusive;
+        return this;
+    }
+
+    constructor(id: string, trigger: IBehaviorElement | IBehaviorElement[], action: IBehaviorElement) {
+        this.id = "Behavior_" + makeNameSafeForUSD(id) + "_" + BehaviorModel.global_id++;
+        this.trigger = trigger;
+        this.action = action;
+    }
+
+    writeTo(_ext: BehaviorExtension, document: USDDocument, writer: USDWriter) {
+        if (!this.trigger || !this.action) return;
+        writer.beginBlock(`def Preliminary_Behavior "${this.id}"`);
+        writer.appendLine(`rel actions = <${this.action.id}>`);
+        writer.appendLine(`uniform bool exclusive = ${this.exclusive}`);
+        let triggerString = "";
+        if (Array.isArray(this.trigger)) {
+            triggerString = "[";
+            for (let i = 0; i < this.trigger.length; i++) {
+                const tr = this.trigger[i];
+                triggerString += "<" + tr.id + ">";
+                if (i + 1 < this.trigger.length) triggerString += ", ";
+            }
+            triggerString += "]";
+        }
+        else
+            triggerString = `<${this.trigger.id}>`;
+
+        writer.appendLine(`rel triggers = ${triggerString} `);
+        writer.appendLine();
+        if (Array.isArray(this.trigger)) {
+            for (const trigger of this.trigger) {
+                trigger.writeTo(document, writer);
+                writer.appendLine();
+            }
+        }
+        else
+            this.trigger.writeTo(document, writer);
+        writer.appendLine();
+        this.action.writeTo(document, writer);
+        writer.closeBlock();
+    }
+}
+
+
+type Target = USDObject | USDObject[] | Object3D | Object3D[];
+
+/** called to resolve target objects to usdz paths */
+function resolve(targetObject: Target, document: USDDocument): string {
+    let result: string = "";
+    if (Array.isArray(targetObject)) {
+        let str = "[ ";
+        for (let i = 0; i < targetObject.length; i++) {
+            let obj = targetObject[i];
+            if (typeof obj === "string")
+                str += obj;
+            else if ( typeof obj === "object") {
+                //@ts-ignore
+                if (obj.isObject3D) {
+                    //@ts-ignore
+                    obj = document.findById(obj.uuid);
+                }
+                const res = (obj as any).getPath?.call(obj) as string;
+                str += res;
+            }
+            if (i + 1 < targetObject.length) str += ", ";
+        }
+        str += " ]";
+        result = str;
+    }
+    else if (typeof targetObject === "object") {
+        //@ts-ignore
+        if (targetObject.isObject3D) {
+            //@ts-ignore
+            targetObject = document.findById(targetObject.uuid);
+        }
+        result = (targetObject as any).getPath?.call(targetObject) as string;
+    }
+
+    // in three there's now a new "Scenes" parent and "Scene" xform that's injected; 
+    // we need to add this to our path here so that we have full paths
+    result = result.replace(document.name, document.name + "/Scenes/Scene");
+    return result;
+}
+
+export class TriggerModel implements IBehaviorElement {
+    static global_id: number = 0;
+
+    id: string;
+    targetId?: string | Target;
+    tokenId?: string;
+    type?: string;
+
+    constructor(targetId?: string | Target, id?: string) {
+        if (targetId) this.targetId = targetId;
+        if (id) this.id = id;
+        else this.id = "Trigger_" + TriggerModel.global_id++;
+    }
+
+    writeTo(document: USDDocument, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Trigger "${this.id}"`);
+        if (this.targetId) {
+            if (typeof this.targetId !== "string") this.targetId = resolve(this.targetId, document);
+            writer.appendLine(`rel affectedObjects = ` + this.targetId);
+        }
+        if (this.tokenId)
+            writer.appendLine(`token info:id = "${this.tokenId}"`);
+        if (this.type)
+            writer.appendLine(`token type = "${this.type}"`);
+        writer.closeBlock();
+    }
+}
+
+export class TriggerBuilder {
+
+    static sceneStartTrigger(): TriggerModel {
+        const trigger = new TriggerModel();
+        trigger.targetId = undefined;
+        trigger.tokenId = "SceneTransition";
+        trigger.type = "enter";
+        return trigger;
+    }
+
+    static tapTrigger(targetObject: Target): TriggerModel {
+        const trigger = new TriggerModel(targetObject);
+        trigger.tokenId = "TapGesture";
+        return trigger;
+    }
+
+    static isTapTrigger(trigger?: TriggerModel) {
+        return trigger?.tokenId === "TapGesture";
+    }
+}
+
+export class GroupActionModel implements IBehaviorElement {
+
+    static global_id: number = 0;
+    static getId(): number {
+        return this.global_id++;
+    }
+
+    id: string;
+    actions: IBehaviorElement[];
+    loops: number = 0;
+    performCount: number = 1;
+    type: string = "serial";
+
+    constructor(id: string, actions: IBehaviorElement[]) {
+        this.id = id;
+        this.actions = actions;
+    }
+
+    addAction(el: IBehaviorElement): GroupActionModel {
+        this.actions.push(el);
+        return this;
+    }
+
+    makeParallel(): GroupActionModel {
+        this.type = "parallel";
+        return this;
+    }
+
+    makeSequence(): GroupActionModel {
+        this.type = "serial";
+        return this;
+    }
+
+    makeLooping() {
+        this.loops = 1;
+        return this;
+    }
+
+    makeRepeat(count: number) {
+        this.performCount = count;
+        return this;
+    }
+
+    writeTo(document: USDDocument, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Action "${this.id}"`);
+        writer.beginArray("rel actions");
+        for (const act of this.actions) {
+            if (!act) continue;
+            writer.appendLine("<" + act.id + ">,");
+        }
+        writer.closeArray();
+        writer.appendLine();
+
+        writer.appendLine(`token info:id = "Group"`);
+        writer.appendLine(`bool loops = ${this.loops} `);
+        writer.appendLine(`int performCount = ${this.performCount} `);
+        writer.appendLine(`token type = "${this.type}"`);
+        writer.appendLine();
+
+        for (const act of this.actions) {
+            if (!act) continue;
+            act.writeTo(document, writer);
+            writer.appendLine();
+        }
+
+        writer.closeBlock();
+    }
+}
+
+export enum MotionType {
+    pop = 0,
+    blink = 1,
+    bounce = 2,
+    flip = 3,
+    float = 4,
+    jiggle = 5,
+    pulse = 6,
+    spin = 7,
+};
+
+export enum Space {
+    Relative = "relative",
+    Absolute = "absolute"
+};
+
+export class ActionModel implements IBehaviorElement {
+
+    private static global_id: number = 0;
+
+    id: string;
+    tokenId?: string;
+    affectedObjects?: string | Target;
+    easeType?: string;;
+    motionType?: string;
+    duration?: number;
+    moveDistance?: number;
+    style?: string;
+    type?: string;
+    front?: Vec3;
+    up?: Vec3;
+    start?: number;
+    animationSpeed?: number;
+    reversed?: boolean;
+    pingPong?: boolean;
+    xFormTarget?: Target | string;
+
+    clone(): ActionModel {
+        const copy = new ActionModel();
+        const id = copy.id;
+        Object.assign(copy, this);
+        copy.id = id;
+        return copy;
+    }
+
+    constructor(affectedObjects?: string | Target, id?: string) {
+        if (affectedObjects) this.affectedObjects = affectedObjects;
+        if (id) this.id = id;
+        /*else if (affectedObjects) {
+            this.id = "Action_" + sanitizeId(affectedObjects);
+        }
+        else */
+        else
+            this.id = "Action";
+        this.id += "_" + ActionModel.global_id++;
+    }
+
+    writeTo(document: USDDocument, writer: USDWriter) {
+        writer.beginBlock(`def Preliminary_Action "${this.id}"`);
+        if (this.affectedObjects) {
+            if (typeof this.affectedObjects !== "string") this.affectedObjects = resolve(this.affectedObjects, document);
+            writer.appendLine('rel affectedObjects = ' + this.affectedObjects);
+        }
+        if (typeof this.duration === "number")
+            writer.appendLine(`double duration = ${this.duration} `);
+        if (this.easeType)
+            writer.appendLine(`token easeType = "${this.easeType}"`);
+        if (this.tokenId)
+            writer.appendLine(`token info:id = "${this.tokenId}"`);
+        if (this.motionType)
+            writer.appendLine(`token motionType = "${this.motionType}"`);
+        if (typeof this.moveDistance === "number")
+            writer.appendLine(`double moveDistance = ${this.moveDistance} `);
+        if (this.style)
+            writer.appendLine(`token style = "${this.style}"`);
+        if (this.type)
+            writer.appendLine(`token type = "${this.type}"`);
+        if (this.front)
+            writer.appendLine(`vector3d front = (${this.front.x}, ${this.front.y}, ${this.front.z})`);
+        if (this.up)
+            writer.appendLine(`vector3d upVector = (${this.up.x}, ${this.up.y}, ${this.up.z})`);
+        if (typeof this.start === "number") {
+            writer.appendLine(`double start = ${this.start} `);
+        }
+        if (typeof this.animationSpeed === "number") {
+            writer.appendLine(`double animationSpeed = ${this.animationSpeed} `);
+        }
+        if (typeof this.reversed === "boolean") {
+            writer.appendLine(`bool reversed = ${this.reversed}`)
+        }
+        if (typeof this.pingPong === "boolean") {
+            writer.appendLine(`bool reverses = ${this.pingPong}`)
+        }
+        if (this.xFormTarget) {
+            if (typeof this.xFormTarget !== "string")
+                this.xFormTarget = resolve(this.xFormTarget, document);
+            writer.appendLine(`rel xformTarget = ${this.xFormTarget}`)
+        }
+        writer.closeBlock();
+    }
+}
+
+class Vec3 {
+    x: number = 0;
+    y: number = 0;
+    z: number = 0;
+
+    constructor(x: number, y: number, z: number) {
+        this.x = x;
+        this.y = y;
+        this.z = z;
+    }
+
+    static get up(): Vec3 {
+        return new Vec3(0, 1, 0);
+    }
+
+    static get right(): Vec3 {
+        return new Vec3(1, 0, 0);
+    }
+
+    static get forward(): Vec3 {
+        return new Vec3(0, 0, 1);
+    }
+
+    static get back(): Vec3 {
+        return new Vec3(0, 0, -1);
+    }
+
+    static get zero(): Vec3 {
+        return new Vec3(0, 0, 0);
+    }
+}
+
+export class ActionBuilder {
+
+    static sequence(...params: IBehaviorElement[]) {
+        const group = new GroupActionModel("group_" + GroupActionModel.getId(), params);
+        return group.makeSequence();
+    }
+
+    static parallel(...params: IBehaviorElement[]) {
+        const group = new GroupActionModel("group_" + GroupActionModel.getId(), params);
+        return group.makeParallel();
+    }
+
+    static fadeAction(targetObject: Target, duration: number, show: boolean): ActionModel {
+        const act = new ActionModel(targetObject);
+        act.tokenId = "Visibility";
+        act.type = show ? "show" : "hide";
+        act.duration = duration;
+
+        act.style = "basic";
+        act.motionType = "none";
+        act.moveDistance = 0;
+        act.easeType = "none";
+        return act;
+    }
+
+    /**
+     * creates an action that plays an animation
+     * @param start offset in seconds!
+     * @param duration in seconds! 0 means play to end
+     */
+    static startAnimationAction(targetObject: Target, start: number, duration: number = 0, animationSpeed: number = 1, reversed: boolean = false, pingPong: boolean = false): IBehaviorElement {
+        const act = new ActionModel(targetObject);
+        act.tokenId = "StartAnimation";
+        act.start = start;
+        // start is time in seconds, the documentation is not right here
+        act.duration = duration;
+        // duration of 0 is play to end
+        act.animationSpeed = animationSpeed;
+        act.reversed = reversed;
+        act.pingPong = pingPong;
+        if (reversed) {
+            act.start -= duration;
+            //console.warn("Reversed animation does currently not work. The resulting file will most likely not playback.", act.id, targetObject);
+        }
+        if (pingPong) {
+            act.pingPong = false;
+            const back = act.clone();
+            back.reversed = !reversed;
+            back.start = act.start;
+            if (back.reversed) {
+                back.start -= duration;
+            }
+            const group = ActionBuilder.sequence(act, back);
+            return group;
+        }
+        return act;
+    }
+
+    static waitAction(duration: number): ActionModel {
+        const act = new ActionModel();
+        act.tokenId = "Wait";
+        act.duration = duration;
+        return act;
+    }
+
+    static lookAtCameraAction(targets: Target, duration: number = 9999999999999, front?: Vec3, up?: Vec3): ActionModel {
+        const act = new ActionModel(targets);
+        act.tokenId = "LookAtCamera";
+        act.duration = duration;
+        act.front = front ?? Vec3.forward;
+        // 0,0,0 is a special case for "free look"
+        // 0,1,0 is for "y-locked look-at"
+        act.up = up ?? Vec3.up;
+        return act;
+    }
+
+    static emphasize(targets: Target, duration: number, motionType: MotionType = MotionType.bounce, moveDistance: number = 1, style: string = "basic") {
+        const act = new ActionModel(targets);
+        act.tokenId = "Emphasize";
+        act.duration = duration;
+        act.style = style ?? "basic";
+        act.motionType = MotionType[motionType]; 
+        act.moveDistance = moveDistance;
+        return act;
+    }
+
+    static transformAction(targets: Target, transformTarget: Target, duration: number, transformType: Space, easeType: string = "inout") {
+        const act = new ActionModel(targets);
+        act.tokenId = "Transform";
+        act.duration = duration;
+        act.type = transformType;
+        act.easeType = easeType;
+        if (Array.isArray(transformTarget)) {
+            console.error("Transform target must not be an array", transformTarget);
+        }
+        act.xFormTarget = transformTarget;
+        return act;
+    }
+
+}
+
+export { Vec3 as USDVec3 }

package/src/engine-components/postprocessing/PostProcessingHandler.ts

@@ -1,4 +1,4 @@
-import { HalfFloatType, sRGBEncoding, WebGLRenderTarget } from "three";
+import { HalfFloatType } from "three";
 import { Context } from "../../engine/engine_setup";
 import { getParam, isMobileDevice } from "../../engine/engine_utils";
 import { BloomEffect, BrightnessContrastEffect, ChromaticAberrationEffect, DepthDownsamplingPass, DepthOfFieldEffect, Effect, EffectComposer, EffectPass, HueSaturationEffect, NormalPass, Pass, PixelationEffect, RenderPass, SelectiveBloomEffect, SSAOEffect, VignetteEffect } from "postprocessing";

package/src/engine-components/ui/BaseUIComponent.ts

@@ -4,7 +4,7 @@ import { Behaviour, GameObject } from "../Component";
 import { EventSystem } from "./EventSystem";
 import { showGizmos } from '../../engine/engine_default_parameters';
 import { AxesHelper, Object3D } from 'three';
-import { IGraphic } from './Interfaces';
+import { ICanvas, IGraphic } from './Interfaces';
 import { ShadowCastingMode } from '../Renderer';
 export const includesDir = "./include";
 
@@ -50,6 +50,12 @@ export class BaseUIComponent extends Behaviour {
         return this._root;
     }
 
+    protected get Canvas() {
+        const cv = this.Root as any as ICanvas;
+        if (cv?.isCanvas) return cv;
+        return null;
+    }
+
     // private _intermediate?: Object3D;
     protected _parentComponent?: BaseUIComponent | null = undefined;
 

package/src/engine-components/ui/Canvas.ts

@@ -3,12 +3,14 @@ import { serializable } from "../../engine/engine_serialization_decorator";
 import { FrameEvent } from "../../engine/engine_setup";
 import { BaseUIComponent, UIRootComponent } from "./BaseUIComponent";
 import { GameObject } from "../Component";
-import { Object3D } from "three";
+import { Matrix4, Object3D } from "three";
 import { RectTransform } from "./RectTransform";
-import { ICanvas } from "./Interfaces";
+import { ICanvas, ILayoutGroup, IRectTransform } from "./Interfaces";
 import { Camera } from "../Camera";
 import { EventSystem } from "./EventSystem";
 import * as ThreeMeshUI from 'three-mesh-ui'
+import { getParam } from "../../engine/engine_utils";
+import { LayoutGroup } from "./Layout";
 
 export enum RenderMode {
     ScreenSpaceOverlay = 0,
@@ -17,8 +19,14 @@ export enum RenderMode {
     Undefined = -1,
 }
 
+const debugLayout = getParam("debuguilayout");
+
 export class Canvas extends UIRootComponent implements ICanvas {
 
+    get isCanvas() {
+        return true;
+    }
+
     get screenspace(): any {
         return this.renderMode !== RenderMode.WorldSpace;
     }
@@ -122,6 +130,10 @@ export class Canvas extends UIRootComponent implements ICanvas {
         super.awake();
     }
 
+    start() {
+        this.onUpdateRenderMode();
+    }
+
     onEnable() {
         super.onEnable();
         this._updateRenderSettingsRoutine = undefined;
@@ -149,6 +161,28 @@ export class Canvas extends UIRootComponent implements ICanvas {
     private _boundRenderSettingsChanged = this.onRenderSettingsChanged.bind(this);
 
     private previousParent: Object3D | null = null;
+    private _lastMatrixWorld: Matrix4 | null = null;
+    private _rectTransforms: IRectTransform[] = [];
+
+    registerTransform(rt: IRectTransform) {
+        this._rectTransforms.push(rt);
+    }
+    unregisterTransform(rt: IRectTransform) {
+        const index = this._rectTransforms.indexOf(rt);
+        if (index !== -1) {
+            this._rectTransforms.splice(index, 1);
+        }
+    }
+
+    private _layoutGroups: Map<Object3D, ILayoutGroup> = new Map();
+    registerLayoutGroup(group: ILayoutGroup) {
+        const obj = group.gameObject;
+        this._layoutGroups.set(obj, group)
+    }
+    unregisterLayoutGroup(group: ILayoutGroup) {
+        const obj = group.gameObject;
+        this._layoutGroups.delete(obj);
+    }
 
     onBeforeRenderRoutine = () => {
         if (this.renderOnTop) {
@@ -158,6 +192,7 @@ export class Canvas extends UIRootComponent implements ICanvas {
         }
         else {
             this.onUpdateRenderMode();
+            this.handleLayoutUpdates();
             // TODO: we might need to optimize this. This is here to make sure the TMUI text clipping matrices are correct. Ideally the text does use onBeforeRender and apply the clipping matrix there so we dont have to force update all the matrices here
             this.shadowComponent?.updateMatrixWorld(true);
             this.shadowComponent?.updateWorldMatrix(true, true);
@@ -171,11 +206,40 @@ export class Canvas extends UIRootComponent implements ICanvas {
             this.context.renderer.autoClear = false;
             this.context.renderer.clearDepth();
             this.onUpdateRenderMode(true);
+            this.handleLayoutUpdates();
             this.shadowComponent?.updateMatrixWorld(true);
+            // this.handleLayoutUpdates();
             EventSystem.ensureUpdateMeshUI(ThreeMeshUI, this.context);
             this.context.renderer.render(this.gameObject, this.context.mainCamera);
             this.context.renderer.autoClear = true;
         }
+        this._lastMatrixWorld?.copy(this.gameObject.matrixWorld);
+    }
+
+    private handleLayoutUpdates() {
+        if (this._lastMatrixWorld === null) {
+            this._lastMatrixWorld = new Matrix4();
+        }
+        const matrixWorldChanged = !this._lastMatrixWorld.equals(this.gameObject.matrixWorld);
+        if (debugLayout && matrixWorldChanged) console.log("Canvas Layout changed", this.context.time.frameCount, this.name);
+
+        // TODO: optimize this, we should only need to update a subhierarchy of the parts where layout has changed
+        let didLog = false;
+        for (const ch of this._rectTransforms) {
+            if (matrixWorldChanged) ch.markDirty();
+            let layout = this._layoutGroups.get(ch.gameObject);
+            if(ch.isDirty && !layout){
+                layout = ch.gameObject.getComponentInParent(LayoutGroup) as LayoutGroup;
+            }
+            if (ch.isDirty || layout?.isDirty) {
+                if (debugLayout && !didLog) {
+                    console.log("CANVAS UPDATE ### " + ch.name + " ##################################### " + this.context.time.frame);
+                    // didLog = true;
+                }
+                layout?.updateLayout();
+                ch.updateTransform();
+            }
+        }
     }
 
     applyRenderSettings() {
@@ -205,6 +269,7 @@ export class Canvas extends UIRootComponent implements ICanvas {
     private _lastWidth: number = -1;
     private _lastHeight: number = -1;
 
+
     private onUpdateRenderMode(force: boolean = false) {
         if (!force) {
             if (this._renderMode === this._activeRenderMode && this._lastWidth === this.context.domWidth && this._lastHeight === this.context.domHeight) {
@@ -241,6 +306,13 @@ export class Canvas extends UIRootComponent implements ICanvas {
                 canvas.quaternion.identity();
 
                 const rect = this.gameObject.getComponent(RectTransform)!;
+                let hasChanged = false;
+                if (rect.sizeDelta.x !== this.context.domWidth) {
+                    hasChanged = true;
+                }
+                if (rect.sizeDelta.y !== this.context.domHeight) {
+                    hasChanged = true;
+                }
 
                 const vFOV = camera.fieldOfView! * Math.PI / 180;
                 const h = 2 * Math.tan(vFOV / 2) * Math.abs(plane);
@@ -248,9 +320,12 @@ export class Canvas extends UIRootComponent implements ICanvas {
                 canvas.scale.y = h / this.context.domHeight;
                 // Set scale.z, otherwise small offsets in screenspace mode have different visual results based on export scale and other settings
                 canvas.scale.z = .01;
-                rect.sizeDelta.x = this.context.domWidth;
-                rect.sizeDelta.y = this.context.domHeight;
-                rect?.markDirty();
+
+                if (hasChanged) {
+                    rect.sizeDelta.x = this.context.domWidth;
+                    rect.sizeDelta.y = this.context.domHeight;
+                    rect?.markDirty();
+                }
 
 
                 // this.context.scene.add(this.gameObject)

package/src/engine-components/ui/Graphic.ts

@@ -9,6 +9,8 @@ import { onChange, scheduleAction } from "./Utils"
 import { GameObject } from '../Component';
 import SimpleStateBehavior from "three-mesh-ui/examples/behaviors/states/SimpleStateBehavior"
 import { Outline } from './Outline';
+import { BehaviorExtension, UsdzBehaviour } from '../../engine-components/export/usdz/extensions/behavior/Behaviour';
+import { USDObject } from '../../engine-components/export/usdz/ThreeUSDZExporter';
 
 const _colorStateObject: { backgroundColor: Color, backgroundOpacity: number } = {
     backgroundColor: new Color(1, 1, 1),

package/src/engine-components/ui/Image.ts

@@ -49,9 +49,9 @@ export class Image extends MaskableGraphic {
             if(this.sprite?.texture?.name === "Knob") {
                 opts.borderRadius = 999;
             }
-            opts.borderColor = new Color(.4, .4, .4);
-            opts.borderOpacity = this.color.alpha;
-            opts.borderWidth = .3;
+            // opts.borderColor = new Color(.4, .4, .4);
+            // opts.borderOpacity = this.color.alpha;
+            // opts.borderWidth = .3;
         }
 
     }