@needle-tools/engine 3.5.1-alpha → 3.5.3-alpha

package/src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts

@@ -6,11 +6,12 @@ import { IPointerClickHandler } from "../../../../ui/PointerEvents";
 import { RegisteredAnimationInfo, UsdzAnimation } from "../Animation";
 import { getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../../../../../engine/engine_three_utils";
 
-import { Object3D, Material, Vector3, Quaternion, AnimationAction } from "three";
-import { USDObject } from "../../ThreeUSDZExporter";
+import { Object3D, Material, Vector3, Quaternion } from "three";
+import { USDDocument, USDObject } from "../../ThreeUSDZExporter";
 
 import { BehaviorExtension, UsdzBehaviour } from "./Behaviour";
-import { ActionBuilder, ActionModel, BehaviorModel, IBehaviorElement, MotionType, Space, TriggerBuilder } from "./BehavioursBuilder";
+import { ActionBuilder, ActionModel, AuralMode, BehaviorModel, IBehaviorElement, MotionType, PlayAction, Space, TriggerBuilder } from "./BehavioursBuilder";
+import { AudioSource } from "../../../../AudioSource";
 
 export class ChangeTransformOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
 
@@ -44,7 +45,7 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
 
         const thisScale = getWorldScale(this.object).clone();
         const targetScale = getWorldScale(this.target).clone();
-        
+
         const dist = thisPos.distanceTo(targetPos);
         const rotDist = thisRot.angleTo(targetRot);
         const scaleDist = thisScale.distanceTo(targetScale);
@@ -63,10 +64,10 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
 
             t01 += this.context.time.deltaTime / this.duration;
             if (t01 > 1) t01 = 1;
-            
+
             // apply ease-in-out
             // https://easings.net/
-            eased =  t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
+            eased = t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
 
             this.targetPos.lerpVectors(thisPos, targetPos, eased);
             this.targetRot.slerpQuaternions(thisRot, targetRot, eased);
@@ -83,7 +84,7 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
     }
 
     private *moveRelative() {
-        
+
         if (!this.target || !this.object) return;
 
         const thisPos = this.object.position.clone();
@@ -107,10 +108,10 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
 
             t01 += this.context.time.deltaTime / this.duration;
             if (t01 > 1) t01 = 1;
-            
+
             // apply ease-in-out
             // https://easings.net/
-            eased =  t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
+            eased = t01 < 0.5 ? 4 * t01 * t01 * t01 : 1 - Math.pow(-2 * t01 + 2, 3) / 2;
 
             this.object.position.lerpVectors(thisPos, this.targetPos, eased);
             this.object.quaternion.slerpQuaternions(thisRot, this.targetRot, eased);
@@ -120,7 +121,7 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
         }
 
         this.coroutine = null;
-    }   
+    }
 
     onPointerClick() {
         if (this.coroutine) this.stopCoroutine(this.coroutine);
@@ -313,11 +314,12 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
                 hideClickedObject = true;
                 targetState = !this.target.visible;
 
-                // TODO check where we have to create the clone; here it doesn't show up
+                if (!this.selfModel.parent || this.selfModel.parent.isEmpty())
+                    USDDocument.createEmptyParent(this.selfModel);
+
                 this.toggleModel = this.selfModel.clone();
                 this.toggleModel.name += "_toggle";
-                if (this.selfModel.parent)
-                    this.selfModel.parent.add(this.toggleModel);
+                this.selfModel.parent!.add(this.toggleModel);
             }
 
             const sequence: ActionModel[] = [];
@@ -344,8 +346,8 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
                     ActionBuilder.sequence(...toggleSequence)
                 ));
 
-                ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name, 
-                    TriggerBuilder.sceneStartTrigger(), 
+                ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name,
+                    TriggerBuilder.sceneStartTrigger(),
                     ActionBuilder.fadeAction(this.toggleModel, 0, false)
                 ));
             }
@@ -363,8 +365,8 @@ export class HideOnStart extends Behaviour implements UsdzBehaviour {
 
     createBehaviours(ext, model, _context) {
         if (model.uuid === this.gameObject.uuid)
-            ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name, 
-                TriggerBuilder.sceneStartTrigger(), 
+            ext.addBehavior(new BehaviorModel("HideOnStart_" + this.gameObject.name,
+                TriggerBuilder.sceneStartTrigger(),
                 ActionBuilder.fadeAction(model, 0, false)
             ));
     }
@@ -394,8 +396,7 @@ export class EmphasizeOnClick extends Behaviour implements UsdzBehaviour {
     createBehaviours(ext, model, _context) {
         if (!this.target) return;
 
-        if (model.uuid === this.gameObject.uuid)
-        {
+        if (model.uuid === this.gameObject.uuid) {
             const emphasize = new BehaviorModel("emphasize " + this.name,
                 TriggerBuilder.tapTrigger(this.gameObject),
                 ActionBuilder.emphasize(this.target, this.duration, this.motionType, undefined, "basic"),
@@ -407,6 +408,82 @@ export class EmphasizeOnClick extends Behaviour implements UsdzBehaviour {
     afterCreateDocument(_ext, _context) { }
 }
 
+export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+
+    @serializable(AudioSource)
+    target?: AudioSource;
+
+    @serializable(URL)
+    clip: string = "";
+
+    @serializable()
+    toggleOnClick: boolean = false;
+
+    onPointerClick() {
+        if (!this.target && !this.clip) return;
+
+        if (!this.target) {
+            
+            const newAudioSource = this.gameObject.addNewComponent(AudioSource);
+            if (newAudioSource) {
+                newAudioSource.spatialBlend = 1;
+                newAudioSource.volume = 1;
+                newAudioSource.loop = false;
+
+                this.target = newAudioSource;
+            }
+        }
+        
+        if (this.target) {
+        
+            if (this.target.isPlaying && this.toggleOnClick) {
+                this.target.stop();
+            }
+            else {
+                if (!this.toggleOnClick && this.target.isPlaying) {
+                    this.target.stop();
+                }
+                if (this.clip) this.target.play(this.clip);
+                else this.target.play();
+            }
+        }
+    }
+    
+    createBehaviours(ext, model, _context) {
+        if (!this.target && !this.clip) return;
+        if (model.uuid === this.gameObject.uuid) {
+
+            const clipUrl = this.clip ? this.clip : this.target ? this.target.clip : undefined;
+            if (!clipUrl) return;
+
+            const playbackTarget = this.target ? this.target.gameObject : this.gameObject;
+            const clipName = clipUrl.split("/").pop();
+            const volume = this.target ? this.target.volume : 1;
+            const auralMode = this.target && this.target.spatialBlend == 0 ? AuralMode.NonSpatial : AuralMode.Spatial;
+            const playClip = new BehaviorModel("playAudio " + this.name,
+                TriggerBuilder.tapTrigger(this.gameObject),
+                ActionBuilder.playAudioAction(playbackTarget, "audio/" + clipName, PlayAction.Play, volume, auralMode),
+            );
+            ext.addBehavior(playClip);
+        }
+    }
+
+    async afterSerialize(_ext, context) {
+        if (!this.target && !this.clip) return;
+        const clipUrl = this.clip ? this.clip : this.target ? this.target.clip : undefined;
+        if (!clipUrl) return;
+        const clipName = clipUrl.split("/").pop();
+
+        const audio = await fetch(this.clip);
+        const audioBlob = await audio.blob();
+        const arrayBuffer = await audioBlob.arrayBuffer();
+
+        const audioData: Uint8Array = new Uint8Array(arrayBuffer)
+
+        context.files["audio/" + clipName] = audioData;
+    }
+}
+
 export class PlayAnimationOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour, UsdzAnimation {
 
     @serializable(Object3D)
@@ -417,7 +494,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
 
     @serializable()
     stateName?: string;
-    
+
     @serializable()
     stateNameAfterPlaying?: string;
 
@@ -486,7 +563,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
 
     createAnimation(ext, model, _context) {
         if (this.target && this.animator) {
-            
+
             const state = this.animator?.runtimeAnimatorController?.findState(this.stateName);
             this.stateAnimationModel = model;
             this.stateAnimation = ext.registerAnimation(this.target, state?.motion.clip);

package/src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts

@@ -74,7 +74,7 @@ function resolve(targetObject: Target, document: USDDocument): string {
             let obj = targetObject[i];
             if (typeof obj === "string")
                 str += obj;
-            else if ( typeof obj === "object") {
+            else if (typeof obj === "object") {
                 //@ts-ignore
                 if (obj.isObject3D) {
                     //@ts-ignore
@@ -237,6 +237,13 @@ export enum Space {
     Absolute = "absolute"
 };
 
+// https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/actions_and_triggers/preliminary_action/multipleperformoperation
+export enum MultiplePerformOperation {
+    Allow = "allow",
+    Ignore = "ignore",
+    Stop = "stop",
+}
+
 export class ActionModel implements IBehaviorElement {
 
     private static global_id: number = 0;
@@ -257,6 +264,10 @@ export class ActionModel implements IBehaviorElement {
     reversed?: boolean;
     pingPong?: boolean;
     xFormTarget?: Target | string;
+    audio?: string;
+    gain?: number;
+    auralMode?: string;
+    multiplePerformOperation?: string;
 
     clone(): ActionModel {
         const copy = new ActionModel();
@@ -319,6 +330,18 @@ export class ActionModel implements IBehaviorElement {
                 this.xFormTarget = resolve(this.xFormTarget, document);
             writer.appendLine(`rel xformTarget = ${this.xFormTarget}`)
         }
+        if (typeof this.audio === "string") {
+            writer.appendLine(`asset audio = @${this.audio}@`);
+        }
+        if (typeof this.gain ==="number") {
+            writer.appendLine(`double gain = ${this.gain}`);
+        }
+        if (typeof this.auralMode === "string") {
+            writer.appendLine(`token auralMode = "${this.auralMode}"`);
+        }
+        if (typeof this.multiplePerformOperation === "string") {
+            writer.appendLine(`token multiplePerformOperation = "${this.multiplePerformOperation}"`);
+        }
         writer.closeBlock();
     }
 }
@@ -355,6 +378,18 @@ class Vec3 {
     }
 }
 
+export enum PlayAction {
+    Play = "play", 
+    Pause = "pause",
+    Stop = "stop",
+}
+
+export enum AuralMode {
+    Spatial = "spatial",
+    NonSpatial = "nonSpatial",
+    Ambient = "ambient",
+}
+
 export class ActionBuilder {
 
     static sequence(...params: IBehaviorElement[]) {
@@ -436,7 +471,7 @@ export class ActionBuilder {
         act.tokenId = "Emphasize";
         act.duration = duration;
         act.style = style ?? "basic";
-        act.motionType = MotionType[motionType]; 
+        act.motionType = MotionType[motionType];
         act.moveDistance = moveDistance;
         return act;
     }
@@ -454,6 +489,17 @@ export class ActionBuilder {
         return act;
     }
 
+    static playAudioAction(targets: Target, audio: string, type: PlayAction = PlayAction.Play, gain: number = 1, auralMode: AuralMode = AuralMode.Spatial) {
+        const act = new ActionModel(targets);
+        act.tokenId = "Audio";
+        act.type = type;
+        act.audio = audio;
+        act.gain = gain;
+        act.auralMode = auralMode;
+        act.multiplePerformOperation = MultiplePerformOperation.Allow;
+        return act;
+    }
+
 }
 
 export { Vec3 as USDVec3 }

package/src/engine-components/ui/BaseUIComponent.ts

@@ -26,6 +26,12 @@ export class BaseUIComponent extends Behaviour {
 
     isRoot() { return this.Root?.gameObject === this.gameObject; }
 
+    get canvas() {
+        const cv = this.Root as any as ICanvas;
+        if (cv?.isCanvas) return cv;
+        return null;
+    }
+
     markDirty() {
         EventSystem.markUIDirty(this.context);
     }
@@ -50,10 +56,9 @@ export class BaseUIComponent extends Behaviour {
         return this._root;
     }
 
+    // TODO: rename to canvas
     protected get Canvas() {
-        const cv = this.Root as any as ICanvas;
-        if (cv?.isCanvas) return cv;
-        return null;
+        return this.canvas;
     }
 
     // private _intermediate?: Object3D;

package/src/engine-components/ui/Canvas.ts

@@ -193,7 +193,7 @@ export class Canvas extends UIRootComponent implements ICanvas {
         this.previousParent = this.gameObject.parent;
         // console.log(this.previousParent?.name + "/" + this.gameObject.name);
 
-        if (this.renderOnTop) {
+        if (this.renderOnTop || this.screenspace) {
             // This is just a test but in reality it should be combined with all world canvases with render on top in one render pass
             this.gameObject.removeFromParent();
         }

package/src/engine-components/ui/EventSystem.ts

@@ -255,7 +255,7 @@ export class EventSystem extends Behaviour {
         if (!hits) return;
         this.lastPointerEvent = args;
 
-        const evt : AfterHandleInputEvent = {
+        const evt: AfterHandleInputEvent = {
             sender: this,
             args: args,
             hasActiveUI: this.currentActiveMeshUIComponents.length > 0,
@@ -433,6 +433,10 @@ export class EventSystem extends Behaviour {
                     }
                 }
 
+                if (comp.onPointerMove) {
+                    comp.onPointerMove(args);
+                }
+
                 if (args.isDown) {
                     if (comp.onPointerDown && !this.raisedPointerDownEvents.includes(comp)) {
                         comp.onPointerDown(args);

package/src/engine-components/ui/Image.ts

@@ -5,13 +5,28 @@ import { MaskableGraphic } from './Graphic';
 
 class Sprite {
     @serializable(Texture)
-    texture?: THREE.Texture;
+    texture: Texture | null = null;
 
     rect?: { width: number, height: number };
 }
 
 export class Image extends MaskableGraphic {
 
+    set image(img: Texture | null) {
+        if (this.sprite)
+            this.sprite.texture = img;
+        else {
+            this.sprite = new Sprite();
+            this.sprite.texture = img;
+        }
+        this.onAfterCreated();
+    }
+    get image(): Texture | null {
+        if (this.sprite)
+            return this.sprite.texture;
+        return null;
+    }
+
     @serializable(Sprite)
     get sprite(): Sprite | undefined {
         return this._sprite;

package/src/engine-components/ui/Interfaces.ts

@@ -1,7 +1,7 @@
 import { Behaviour } from "../Component";
 import { IComponent } from "../../engine/engine_types";
 
-export interface ICanvas {
+export interface ICanvas extends IComponent {
     get isCanvas(): boolean;
     get screenspace(): boolean;
     registerTransform(rt: IRectTransform);

package/src/engine-components/ui/Layout.ts

@@ -250,6 +250,8 @@ export abstract class HorizontalOrVerticalLayoutGroup extends LayoutGroup {
             const rt = GameObject.getComponent(ch, RectTransform);
             if (rt?.activeAndEnabled) {
                 rt.pivot?.set(.5, .5);
+                rt.anchorMin.set(0, 1);
+                rt.anchorMax.set(0, 1);
                 // Horizontal padding
                 const x = totalWidth * .5 + leftOffset * .5;
                 if (rt.anchoredPosition.x !== x)

package/src/engine-components/ui/PointerEvents.ts

@@ -11,12 +11,22 @@ export class PointerEventData implements IInputEventArgs {
     // TODO: should we make this a getter and return the input used state instead? -> this.context.getPointerUsed(this.pointerId);
     used: boolean = false;
 
-    Use() {
+    use() {
         this.used = true;
         if (this.pointerId !== undefined)
             this.input.setPointerUsed(this.pointerId);
     }
 
+    stopPropagation() {
+        this.event?.stopImmediatePropagation();
+    }
+
+    /**@deprecated use use() */
+    Use() {
+        this.use();
+    }
+
+    /**@deprecated use stopPropagation() */
     StopPropagation() {
         this.event?.stopImmediatePropagation();
     }
@@ -52,6 +62,10 @@ export interface IPointerEnterHandler {
     onPointerEnter?(args: PointerEventData);
 }
 
+export interface IPointerMoveHandler {
+    onPointerMove?(args: PointerEventData);
+}
+
 export interface IPointerExitHandler {
     onPointerExit?(args: PointerEventData);
 }
@@ -61,4 +75,4 @@ export interface IPointerClickHandler {
 }
 
 export interface IPointerEventHandler extends IPointerDownHandler,
-    IPointerUpHandler, IPointerEnterHandler, IPointerExitHandler, IPointerClickHandler { }
+    IPointerUpHandler, IPointerEnterHandler, IPointerMoveHandler, IPointerExitHandler, IPointerClickHandler { }

package/src/engine-components/ui/RaycastUtils.ts

@@ -20,7 +20,12 @@ export class UIRaycastUtils {
     };
 
     static isInteractable(obj: THREE.Object3D, out?: { canvasGroup?: ICanvasGroup, graphic?: IGraphic }): boolean {
-        if(obj === null || obj === undefined || !obj.visible) return false;
+        // reset state
+        if (out) {
+            out.canvasGroup = undefined;
+            out.graphic = undefined;
+        }
+        if (obj === null || obj === undefined || !obj.visible) return false;
 
         obj = this.getObject(obj);
 

package/src/engine-components/ui/RectTransform.ts

@@ -31,6 +31,9 @@ const tempQuaternion = new Quaternion();
 
 export class RectTransform extends BaseUIComponent implements IRectTransform, IRectTransformChangedReceiver {
 
+    get parent() {
+        return this._parentRectTransform;
+    }
     offset: number = .01;
 
     // @serializable(Object3D)
@@ -51,24 +54,21 @@ export class RectTransform extends BaseUIComponent implements IRectTransform, IR
         this._anchoredPosition = value;
     }
 
-    @serializable(Rect)
-    private rect?: Rect; // TODO: should we use the rect or sizeDelta?
-
     @serializable(Vector2)
-    sizeDelta!: Vector2;
+    sizeDelta: Vector2 = new Vector2(100, 100);
 
     @serializable(Vector2)
-    pivot?: Vector2;
+    pivot: Vector2 = new Vector2(.5, .5);
 
     @serializable(Vector2)
-    anchorMin!: Vector2;
+    anchorMin: Vector2 = new Vector2(0, 0);
     @serializable(Vector2)
-    anchorMax!: Vector2;
+    anchorMax: Vector2 = new Vector2(1, 1);
 
     @serializable(Vector2)
-    offsetMin!: Vector2;
+    offsetMin: Vector2 = new Vector2(0, 0);
     @serializable(Vector2)
-    offsetMax!: Vector2;
+    offsetMax: Vector2 = new Vector2(0, 0);
 
     get width() {
         if (this.anchorMin.x !== this.anchorMax.x) {
@@ -301,8 +301,8 @@ export class RectTransform extends BaseUIComponent implements IRectTransform, IR
         // })
 
         const opts = {
-            width: this.rect!.width,
-            height: this.rect!.height,// * this.context.mainCameraComponent!.aspect,
+            width: this.sizeDelta!.x,
+            height: this.sizeDelta!.y,// * this.context.mainCameraComponent!.aspect,
             offset: this.offset,
             backgroundOpacity: 0,
             borderWidth: 0, // if we dont specify width here a border will automatically propagated to child blocks

package/src/engine-components/ui/Text.ts

@@ -40,8 +40,6 @@ export enum FontStyle {
 
 export class Text extends Graphic {
 
-    @serializable(Canvas)
-    canvas?: Canvas;
     @serializable()
     alignment: TextAnchor = TextAnchor.UpperLeft;
     @serializable()
@@ -334,7 +332,7 @@ export class Text extends Graphic {
     private * renderOnTopCoroutine() {
         if (!this.canvas) return;
         const updatedRendering: boolean[] = [];
-        const canvas = this.canvas;
+        const canvas = this.canvas as Canvas;
         const settings = {
             renderOnTop: canvas.renderOnTop,
             depthWrite: canvas.depthWrite,