@needle-tools/engine 3.5.2-alpha → 3.5.3-alpha

package/src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts

@@ -6,11 +6,12 @@ import { IPointerClickHandler } from "../../../../ui/PointerEvents";
 import { RegisteredAnimationInfo, UsdzAnimation } from "../Animation";
 import { getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../../../../../engine/engine_three_utils";
 
-import { Object3D, Material, Vector3, Quaternion, AnimationAction } from "three";
+import { Object3D, Material, Vector3, Quaternion } from "three";
 import { USDDocument, USDObject } from "../../ThreeUSDZExporter";
 
 import { BehaviorExtension, UsdzBehaviour } from "./Behaviour";
-import { ActionBuilder, ActionModel, BehaviorModel, IBehaviorElement, MotionType, Space, TriggerBuilder } from "./BehavioursBuilder";
+import { ActionBuilder, ActionModel, AuralMode, BehaviorModel, IBehaviorElement, MotionType, PlayAction, Space, TriggerBuilder } from "./BehavioursBuilder";
+import { AudioSource } from "../../../../AudioSource";
 
 export class ChangeTransformOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
 
@@ -407,6 +408,82 @@ export class EmphasizeOnClick extends Behaviour implements UsdzBehaviour {
     afterCreateDocument(_ext, _context) { }
 }
 
+export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour {
+
+    @serializable(AudioSource)
+    target?: AudioSource;
+
+    @serializable(URL)
+    clip: string = "";
+
+    @serializable()
+    toggleOnClick: boolean = false;
+
+    onPointerClick() {
+        if (!this.target && !this.clip) return;
+
+        if (!this.target) {
+            
+            const newAudioSource = this.gameObject.addNewComponent(AudioSource);
+            if (newAudioSource) {
+                newAudioSource.spatialBlend = 1;
+                newAudioSource.volume = 1;
+                newAudioSource.loop = false;
+
+                this.target = newAudioSource;
+            }
+        }
+        
+        if (this.target) {
+        
+            if (this.target.isPlaying && this.toggleOnClick) {
+                this.target.stop();
+            }
+            else {
+                if (!this.toggleOnClick && this.target.isPlaying) {
+                    this.target.stop();
+                }
+                if (this.clip) this.target.play(this.clip);
+                else this.target.play();
+            }
+        }
+    }
+    
+    createBehaviours(ext, model, _context) {
+        if (!this.target && !this.clip) return;
+        if (model.uuid === this.gameObject.uuid) {
+
+            const clipUrl = this.clip ? this.clip : this.target ? this.target.clip : undefined;
+            if (!clipUrl) return;
+
+            const playbackTarget = this.target ? this.target.gameObject : this.gameObject;
+            const clipName = clipUrl.split("/").pop();
+            const volume = this.target ? this.target.volume : 1;
+            const auralMode = this.target && this.target.spatialBlend == 0 ? AuralMode.NonSpatial : AuralMode.Spatial;
+            const playClip = new BehaviorModel("playAudio " + this.name,
+                TriggerBuilder.tapTrigger(this.gameObject),
+                ActionBuilder.playAudioAction(playbackTarget, "audio/" + clipName, PlayAction.Play, volume, auralMode),
+            );
+            ext.addBehavior(playClip);
+        }
+    }
+
+    async afterSerialize(_ext, context) {
+        if (!this.target && !this.clip) return;
+        const clipUrl = this.clip ? this.clip : this.target ? this.target.clip : undefined;
+        if (!clipUrl) return;
+        const clipName = clipUrl.split("/").pop();
+
+        const audio = await fetch(this.clip);
+        const audioBlob = await audio.blob();
+        const arrayBuffer = await audioBlob.arrayBuffer();
+
+        const audioData: Uint8Array = new Uint8Array(arrayBuffer)
+
+        context.files["audio/" + clipName] = audioData;
+    }
+}
+
 export class PlayAnimationOnClick extends Behaviour implements IPointerClickHandler, UsdzBehaviour, UsdzAnimation {
 
     @serializable(Object3D)

package/src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts

@@ -237,6 +237,13 @@ export enum Space {
     Absolute = "absolute"
 };
 
+// https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/actions_and_triggers/preliminary_action/multipleperformoperation
+export enum MultiplePerformOperation {
+    Allow = "allow",
+    Ignore = "ignore",
+    Stop = "stop",
+}
+
 export class ActionModel implements IBehaviorElement {
 
     private static global_id: number = 0;
@@ -257,6 +264,10 @@ export class ActionModel implements IBehaviorElement {
     reversed?: boolean;
     pingPong?: boolean;
     xFormTarget?: Target | string;
+    audio?: string;
+    gain?: number;
+    auralMode?: string;
+    multiplePerformOperation?: string;
 
     clone(): ActionModel {
         const copy = new ActionModel();
@@ -319,6 +330,18 @@ export class ActionModel implements IBehaviorElement {
                 this.xFormTarget = resolve(this.xFormTarget, document);
             writer.appendLine(`rel xformTarget = ${this.xFormTarget}`)
         }
+        if (typeof this.audio === "string") {
+            writer.appendLine(`asset audio = @${this.audio}@`);
+        }
+        if (typeof this.gain ==="number") {
+            writer.appendLine(`double gain = ${this.gain}`);
+        }
+        if (typeof this.auralMode === "string") {
+            writer.appendLine(`token auralMode = "${this.auralMode}"`);
+        }
+        if (typeof this.multiplePerformOperation === "string") {
+            writer.appendLine(`token multiplePerformOperation = "${this.multiplePerformOperation}"`);
+        }
         writer.closeBlock();
     }
 }
@@ -355,6 +378,18 @@ class Vec3 {
     }
 }
 
+export enum PlayAction {
+    Play = "play", 
+    Pause = "pause",
+    Stop = "stop",
+}
+
+export enum AuralMode {
+    Spatial = "spatial",
+    NonSpatial = "nonSpatial",
+    Ambient = "ambient",
+}
+
 export class ActionBuilder {
 
     static sequence(...params: IBehaviorElement[]) {
@@ -454,6 +489,17 @@ export class ActionBuilder {
         return act;
     }
 
+    static playAudioAction(targets: Target, audio: string, type: PlayAction = PlayAction.Play, gain: number = 1, auralMode: AuralMode = AuralMode.Spatial) {
+        const act = new ActionModel(targets);
+        act.tokenId = "Audio";
+        act.type = type;
+        act.audio = audio;
+        act.gain = gain;
+        act.auralMode = auralMode;
+        act.multiplePerformOperation = MultiplePerformOperation.Allow;
+        return act;
+    }
+
 }
 
 export { Vec3 as USDVec3 }

package/src/engine-components/ui/BaseUIComponent.ts

@@ -26,6 +26,12 @@ export class BaseUIComponent extends Behaviour {
 
     isRoot() { return this.Root?.gameObject === this.gameObject; }
 
+    get canvas() {
+        const cv = this.Root as any as ICanvas;
+        if (cv?.isCanvas) return cv;
+        return null;
+    }
+
     markDirty() {
         EventSystem.markUIDirty(this.context);
     }
@@ -50,10 +56,9 @@ export class BaseUIComponent extends Behaviour {
         return this._root;
     }
 
+    // TODO: rename to canvas
     protected get Canvas() {
-        const cv = this.Root as any as ICanvas;
-        if (cv?.isCanvas) return cv;
-        return null;
+        return this.canvas;
     }
 
     // private _intermediate?: Object3D;

package/src/engine-components/ui/Canvas.ts

@@ -193,7 +193,7 @@ export class Canvas extends UIRootComponent implements ICanvas {
         this.previousParent = this.gameObject.parent;
         // console.log(this.previousParent?.name + "/" + this.gameObject.name);
 
-        if (this.renderOnTop) {
+        if (this.renderOnTop || this.screenspace) {
             // This is just a test but in reality it should be combined with all world canvases with render on top in one render pass
             this.gameObject.removeFromParent();
         }

package/src/engine-components/ui/EventSystem.ts

@@ -255,7 +255,7 @@ export class EventSystem extends Behaviour {
         if (!hits) return;
         this.lastPointerEvent = args;
 
-        const evt : AfterHandleInputEvent = {
+        const evt: AfterHandleInputEvent = {
             sender: this,
             args: args,
             hasActiveUI: this.currentActiveMeshUIComponents.length > 0,
@@ -433,6 +433,10 @@ export class EventSystem extends Behaviour {
                     }
                 }
 
+                if (comp.onPointerMove) {
+                    comp.onPointerMove(args);
+                }
+
                 if (args.isDown) {
                     if (comp.onPointerDown && !this.raisedPointerDownEvents.includes(comp)) {
                         comp.onPointerDown(args);

package/src/engine-components/ui/Image.ts

@@ -5,13 +5,28 @@ import { MaskableGraphic } from './Graphic';
 
 class Sprite {
     @serializable(Texture)
-    texture?: THREE.Texture;
+    texture: Texture | null = null;
 
     rect?: { width: number, height: number };
 }
 
 export class Image extends MaskableGraphic {
 
+    set image(img: Texture | null) {
+        if (this.sprite)
+            this.sprite.texture = img;
+        else {
+            this.sprite = new Sprite();
+            this.sprite.texture = img;
+        }
+        this.onAfterCreated();
+    }
+    get image(): Texture | null {
+        if (this.sprite)
+            return this.sprite.texture;
+        return null;
+    }
+
     @serializable(Sprite)
     get sprite(): Sprite | undefined {
         return this._sprite;

package/src/engine-components/ui/Interfaces.ts

@@ -1,7 +1,7 @@
 import { Behaviour } from "../Component";
 import { IComponent } from "../../engine/engine_types";
 
-export interface ICanvas {
+export interface ICanvas extends IComponent {
     get isCanvas(): boolean;
     get screenspace(): boolean;
     registerTransform(rt: IRectTransform);

package/src/engine-components/ui/Layout.ts

@@ -250,6 +250,8 @@ export abstract class HorizontalOrVerticalLayoutGroup extends LayoutGroup {
             const rt = GameObject.getComponent(ch, RectTransform);
             if (rt?.activeAndEnabled) {
                 rt.pivot?.set(.5, .5);
+                rt.anchorMin.set(0, 1);
+                rt.anchorMax.set(0, 1);
                 // Horizontal padding
                 const x = totalWidth * .5 + leftOffset * .5;
                 if (rt.anchoredPosition.x !== x)

package/src/engine-components/ui/PointerEvents.ts

@@ -11,12 +11,22 @@ export class PointerEventData implements IInputEventArgs {
     // TODO: should we make this a getter and return the input used state instead? -> this.context.getPointerUsed(this.pointerId);
     used: boolean = false;
 
-    Use() {
+    use() {
         this.used = true;
         if (this.pointerId !== undefined)
             this.input.setPointerUsed(this.pointerId);
     }
 
+    stopPropagation() {
+        this.event?.stopImmediatePropagation();
+    }
+
+    /**@deprecated use use() */
+    Use() {
+        this.use();
+    }
+
+    /**@deprecated use stopPropagation() */
     StopPropagation() {
         this.event?.stopImmediatePropagation();
     }
@@ -52,6 +62,10 @@ export interface IPointerEnterHandler {
     onPointerEnter?(args: PointerEventData);
 }
 
+export interface IPointerMoveHandler {
+    onPointerMove?(args: PointerEventData);
+}
+
 export interface IPointerExitHandler {
     onPointerExit?(args: PointerEventData);
 }
@@ -61,4 +75,4 @@ export interface IPointerClickHandler {
 }
 
 export interface IPointerEventHandler extends IPointerDownHandler,
-    IPointerUpHandler, IPointerEnterHandler, IPointerExitHandler, IPointerClickHandler { }
+    IPointerUpHandler, IPointerEnterHandler, IPointerMoveHandler, IPointerExitHandler, IPointerClickHandler { }

package/src/engine-components/ui/RaycastUtils.ts

@@ -20,7 +20,12 @@ export class UIRaycastUtils {
     };
 
     static isInteractable(obj: THREE.Object3D, out?: { canvasGroup?: ICanvasGroup, graphic?: IGraphic }): boolean {
-        if(obj === null || obj === undefined || !obj.visible) return false;
+        // reset state
+        if (out) {
+            out.canvasGroup = undefined;
+            out.graphic = undefined;
+        }
+        if (obj === null || obj === undefined || !obj.visible) return false;
 
         obj = this.getObject(obj);
 

package/src/engine-components/ui/RectTransform.ts

@@ -31,6 +31,9 @@ const tempQuaternion = new Quaternion();
 
 export class RectTransform extends BaseUIComponent implements IRectTransform, IRectTransformChangedReceiver {
 
+    get parent() {
+        return this._parentRectTransform;
+    }
     offset: number = .01;
 
     // @serializable(Object3D)
@@ -51,24 +54,21 @@ export class RectTransform extends BaseUIComponent implements IRectTransform, IR
         this._anchoredPosition = value;
     }
 
-    @serializable(Rect)
-    private rect?: Rect; // TODO: should we use the rect or sizeDelta?
-
     @serializable(Vector2)
-    sizeDelta!: Vector2;
+    sizeDelta: Vector2 = new Vector2(100, 100);
 
     @serializable(Vector2)
-    pivot?: Vector2;
+    pivot: Vector2 = new Vector2(.5, .5);
 
     @serializable(Vector2)
-    anchorMin!: Vector2;
+    anchorMin: Vector2 = new Vector2(0, 0);
     @serializable(Vector2)
-    anchorMax!: Vector2;
+    anchorMax: Vector2 = new Vector2(1, 1);
 
     @serializable(Vector2)
-    offsetMin!: Vector2;
+    offsetMin: Vector2 = new Vector2(0, 0);
     @serializable(Vector2)
-    offsetMax!: Vector2;
+    offsetMax: Vector2 = new Vector2(0, 0);
 
     get width() {
         if (this.anchorMin.x !== this.anchorMax.x) {
@@ -301,8 +301,8 @@ export class RectTransform extends BaseUIComponent implements IRectTransform, IR
         // })
 
         const opts = {
-            width: this.rect!.width,
-            height: this.rect!.height,// * this.context.mainCameraComponent!.aspect,
+            width: this.sizeDelta!.x,
+            height: this.sizeDelta!.y,// * this.context.mainCameraComponent!.aspect,
             offset: this.offset,
             backgroundOpacity: 0,
             borderWidth: 0, // if we dont specify width here a border will automatically propagated to child blocks

package/src/engine-components/ui/Text.ts

@@ -40,8 +40,6 @@ export enum FontStyle {
 
 export class Text extends Graphic {
 
-    @serializable(Canvas)
-    canvas?: Canvas;
     @serializable()
     alignment: TextAnchor = TextAnchor.UpperLeft;
     @serializable()
@@ -334,7 +332,7 @@ export class Text extends Graphic {
     private * renderOnTopCoroutine() {
         if (!this.canvas) return;
         const updatedRendering: boolean[] = [];
-        const canvas = this.canvas;
+        const canvas = this.canvas as Canvas;
         const settings = {
             renderOnTop: canvas.renderOnTop,
             depthWrite: canvas.depthWrite,