@needle-tools/engine 4.14.0 → 4.15.0-next.cecd8e7

package/src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts

@@ -13,12 +13,11 @@ import { AudioSource } from "../../../../AudioSource.js";
 import { Behaviour, GameObject } from "../../../../Component.js";
 import { Rigidbody } from "../../../../RigidBody.js";
 import type { IPointerClickHandler, PointerEventData } from "../../../../ui/PointerEvents.js";
-
-import { makeNameSafeForUSD,USDDocument, USDObject, USDZExporterContext } from "../../ThreeUSDZExporter.js";
+import { makeNameSafeForUSD, USDDocument, USDObject, USDZExporterContext } from "../../ThreeUSDZExporter.js";
 import { AnimationExtension, RegisteredAnimationInfo, type UsdzAnimation } from "../Animation.js";
 import { AudioExtension } from "./AudioExtension.js";
 import type { BehaviorExtension, UsdzBehaviour } from "./Behaviour.js";
-import { ActionBuilder, ActionModel, BehaviorModel, EmphasizeActionMotionType,GroupActionModel,type IBehaviorElement, Target, TriggerBuilder } from "./BehavioursBuilder.js";
+import { ActionBuilder, ActionModel, BehaviorModel, EmphasizeActionMotionType, GroupActionModel, type IBehaviorElement, Target, TriggerBuilder } from "./BehavioursBuilder.js";
 
 const debug = getParam("debugusdzbehaviours");
 
@@ -57,6 +56,22 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
     private targetRot = new Quaternion();
     private targetScale = new Vector3();
 
+    onEnable(): void {
+        this.context.accessibility.updateElement(this, {
+            role: "button",
+            label: "Move " + (this.object?.name || "object") + " to " + (this.target?.name || "target") + " on click",
+            hidden: false,
+        })
+    }
+    onDisable(): void {
+        this.context.accessibility.updateElement(this, {
+            hidden: true,
+        });
+    }
+    onDestroy(): void {
+        this.context.accessibility.removeElement(this);
+    }
+
     onPointerEnter() {
         this.context.input.setCursor("pointer");
     }
@@ -68,8 +83,8 @@ export class ChangeTransformOnClick extends Behaviour implements IPointerClickHa
 
         const rbs = this.object?.getComponentsInChildren(Rigidbody);
 
-        if (rbs){
-            for (const rb of rbs) {	
+        if (rbs) {
+            for (const rb of rbs) {
                 rb.resetVelocities();
                 rb.resetForcesAndTorques();
             }
@@ -227,6 +242,22 @@ export class ChangeMaterialOnClick extends Behaviour implements IPointerClickHan
         }
     }
 
+    onEnable(): void {
+        this.context.accessibility.updateElement(this, {
+            role: "button",
+            label: "Change material to " + (this.variantMaterial?.name || "unknown material"),
+            hidden: false,
+        })
+    }
+    onDisable(): void {
+        this.context.accessibility.updateElement(this, {
+            hidden: true,
+        });
+    }
+    onDestroy(): void {
+        this.context.accessibility.removeElement(this);
+    }
+
     onPointerEnter(_args: PointerEventData) {
         this.context.input.setCursor("pointer");
     }
@@ -427,7 +458,7 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
 
         if (!this.toggleOnClick && this.hideSelf)
             this.gameObject.visible = false;
-      
+
         if (this.target)
             this.target.visible = this.toggleOnClick ? !this.target.visible : this.targetState;
     }
@@ -453,7 +484,7 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
 
     beforeCreateDocument() {
         if (!this.target) return;
-      
+
         // need to cache on the object itself, because otherwise different actions would override each other's visibility state
         // TODO would probably be better to have this somewhere on the exporter, not on this component
         if (this.gameObject[SetActiveOnClick.wasVisible] === undefined)
@@ -514,7 +545,7 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
                 // It's much easier to reason about nested actions when we're not duplicating tons of hierarchy...
                 // We can probably only do a shallow clone when the tapped object has geometry of its own, otherwise
                 // we end up with nothing to tap on.
-              
+
                 // Option A: we deep clone ourselves. This makes hierarchical cases and nested behaviours really complex.
                 // We do this currently when the object doesn't have any geometry.
                 if (!this.selfModelClone.geometry) {
@@ -533,11 +564,11 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
                         clone.name += "_toggle" + (SetActiveOnClick.clonedToggleIndex++);
                         originalModel.add(clone);
                         this.gameObject[SetActiveOnClick.toggleClone] = clone;
-                      
+
                         console.warn("USDZExport: Toggle " + this.gameObject.name + " doesn't have geometry. It will be deep cloned and nested behaviours will likely not work.");
                     }
                     const clonedSelfModel = this.gameObject[SetActiveOnClick.toggleClone];
-                  
+
                     if (!this.gameObject[SetActiveOnClick.reverseToggleClone]) {
                         const clone = this.selfModelClone.clone();
                         clone.setMatrix(new Matrix4());
@@ -589,7 +620,7 @@ export class SetActiveOnClick extends Behaviour implements IPointerClickHandler,
                     TriggerBuilder.tapTrigger(selfModel),
                     ActionBuilder.parallel(...toggleSequence)
                 ));
-              
+
                 const reverseSequence: ActionModel[] = [];
                 reverseSequence.push(ActionBuilder.fadeAction(this.toggleModel, 0, false));
                 reverseSequence.push(ActionBuilder.fadeAction(selfModel, 0, true));
@@ -680,7 +711,7 @@ export class HideOnStart extends Behaviour implements UsdzBehaviour {
     }
 
     private wasVisible: boolean = false;
-  
+
     beforeCreateDocument() {
         this.wasVisible = GameObject.isActiveSelf(this.gameObject);
     }
@@ -713,6 +744,22 @@ export class EmphasizeOnClick extends Behaviour implements UsdzBehaviour {
     @serializable()
     motionType: EmphasizeActionMotionType = "bounce";
 
+    onEnable(): void {
+        this.context.accessibility.updateElement(this, {
+            role: "button",
+            label: "Emphasize " + this.target?.name + " on click",
+            hidden: false,
+        })
+    }
+    onDisable(): void {
+        this.context.accessibility.updateElement(this, {
+            hidden: true,
+        });
+    }
+    onDestroy(): void {
+        this.context.accessibility.removeElement(this);
+    }
+
     beforeCreateDocument() { }
 
     createBehaviours(ext, model, _context) {
@@ -775,6 +822,21 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
         }
     }
 
+    onEnable(): void {
+        this.context.accessibility.updateElement(this, {
+            role: "button",
+            label: "Play audio: " + (this.clip || this.target?.clip || "unknown clip"),
+            hidden: false,
+        })
+    }
+    onDisable(): void {
+        this.context.accessibility.updateElement(this, {
+            hidden: true,
+        });
+    }
+    onDestroy(): void {
+        this.context.accessibility.removeElement(this);
+    }
 
     onPointerEnter() {
         this.context.input.setCursor("pointer");
@@ -816,7 +878,7 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
             const clipName = AudioExtension.getName(clipUrl);
             const volume = this.target ? this.target.volume : 1;
             const auralMode = this.target && this.target.spatialBlend == 0 ? "nonSpatial" : "spatial";
-          
+
             // This checks if any child is clickable – if yes, the tap trigger is added; if not, we omit it.
             let anyChildHasGeometry = false;
             this.gameObject.traverse(c => {
@@ -825,7 +887,7 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
             // Workaround: seems iOS often simply doesn't play audio on scene start when this is NOT present.
             // unclear why, but having a useless tap action (nothing to tap on) "fixes" it.
             anyChildHasGeometry = true;
-          
+
             const audioClip = ext.addAudioClip(clipUrl);
             // const stopAction: IBehaviorElement = ActionBuilder.playAudioAction(playbackTarget, audioClip, "stop", volume, auralMode);
             let playAction: IBehaviorElement = ActionBuilder.playAudioAction(playbackTarget, audioClip, "play", volume, auralMode);
@@ -834,8 +896,7 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
 
             const behaviorName = (this.name ? "_" + this.name : "");
 
-            if (anyChildHasGeometry && this.trigger === "tap")
-            {
+            if (anyChildHasGeometry && this.trigger === "tap") {
                 // does not seem to work in iOS / QuickLook...
                 // TODO use play "type" which can be start/stop/pause
                 if (this.toggleOnClick) (playAction as ActionModel).multiplePerformOperation = "stop";
@@ -901,9 +962,25 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
 
     private get target() { return this.animator?.gameObject || this.animation?.gameObject }
 
+    onEnable(): void {
+        this.context.accessibility.updateElement(this, {
+            role: "button",
+            label: "Plays animation " + (this.stateName || "") + " on " + (this.target ? this.target.name : ""),
+            hidden: false
+        });
+    }
+    onDisable(): void {
+        this.context.accessibility.updateElement(this, {
+            hidden: true,
+        });
+    }
+    onDestroy(): void {
+        this.context.accessibility.removeElement(this);
+    }
 
     onPointerEnter() {
         this.context.input.setCursor("pointer");
+        this.context.accessibility.hover(this, "Click to play animation " + (this.stateName || "") + " on " + (this.target ? this.target.name : ""));
     }
     onPointerExit() {
         this.context.input.unsetCursor("pointer");
@@ -912,6 +989,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
         args.use();
         if (!this.target) return;
         if (this.stateName) {
+            this.context.accessibility.focus(this);
             // TODO this is currently quite annoying to use,
             // as for the web we use the Animator component and its states directly,
             // while in QuickLook we use explicit animations / states.
@@ -923,8 +1001,8 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
 
     private stateAnimationModel: any;
 
-    private animationSequence? = new Array<RegisteredAnimationInfo>();
-    private animationLoopAfterSequence? = new Array<RegisteredAnimationInfo>();
+    private animationSequence?= new Array<RegisteredAnimationInfo>();
+    private animationLoopAfterSequence?= new Array<RegisteredAnimationInfo>();
     private randomOffsetNormalized: number = 0;
 
     createBehaviours(_ext: BehaviorExtension, model: USDObject, _context: USDZExporterContext) {
@@ -950,9 +1028,9 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
     afterCreateDocument(ext: BehaviorExtension, context: USDZExporterContext) {
         if ((this.animationSequence === undefined && this.animationLoopAfterSequence === undefined) || !this.stateAnimationModel) return;
         if (!this.target) return;
-      
+
         const document = context.document;
-      
+
         // check if the AnimationExtension has been attached and what data it has for the current object
         const animationExt = context.extensions.find(ext => ext instanceof AnimationExtension) as AnimationExtension;
         if (!animationExt) return;
@@ -966,7 +1044,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
         if (requiresExclusivePlayback) {
             if (isDevEnvironment())
                 console.warn("Setting exclusive playback for " + this.target.name + "@" + this.stateName + " because it has " + animationExt.getClipCount(this.target) + " animations. This works around QuickLook bug FB13410767.");
-          
+
             PlayAnimationOnClick.rootsWithExclusivePlayback.add(this.target);
         }
 
@@ -986,7 +1064,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
                     this.trigger == "tap" ? TriggerBuilder.tapTrigger(this.selfModel) : TriggerBuilder.sceneStartTrigger(),
                     sequence
                 );
-              
+
                 // See comment above for why exclusive playback is currently required when playing multiple animations on the same root.
                 if (requiresExclusivePlayback)
                     playAnimationOnTap.makeExclusive(true);
@@ -1008,8 +1086,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
 
         const sequence = ActionBuilder.sequence();
 
-        if (animationSequence && animationSequence.length > 0)
-        {
+        if (animationSequence && animationSequence.length > 0) {
             for (const anim of animationSequence) {
                 sequence.addAction(getOrCacheAction(model, anim));
             }
@@ -1033,11 +1110,10 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
         return sequence;
     }
 
-    static getAndRegisterAnimationSequences(ext: AnimationExtension, target: GameObject, stateName?: string):
-    { 
-        animationSequence: Array<RegisteredAnimationInfo>, 
+    static getAndRegisterAnimationSequences(ext: AnimationExtension, target: GameObject, stateName?: string): {
+        animationSequence: Array<RegisteredAnimationInfo>,
         animationLoopAfterSequence: Array<RegisteredAnimationInfo>,
-        randomTimeOffset: number, 
+        randomTimeOffset: number,
     } | undefined {
 
         if (!target) return undefined;
@@ -1055,14 +1131,14 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
         let animationLoopAfterSequence: Array<RegisteredAnimationInfo> = [];
 
         if (animation) {
-            const anim = ext.registerAnimation(target, animation.clip); 
-            if (anim) { 
+            const anim = ext.registerAnimation(target, animation.clip);
+            if (anim) {
                 if (animation.loop)
                     animationLoopAfterSequence.push(anim);
                 else
                     animationSequence.push(anim);
             }
-          
+
             let randomTimeOffset = 0;
             if (animation.minMaxOffsetNormalized) {
                 const from = animation.minMaxOffsetNormalized.x;
@@ -1156,7 +1232,7 @@ export class PlayAnimationOnClick extends Behaviour implements IPointerClickHand
                 if (lastClip) {
                     let clipCopy: AnimationClip | undefined;
                     if (ext.holdClipMap.has(lastClip)) {
-                        clipCopy = ext.holdClipMap.get(lastClip);  
+                        clipCopy = ext.holdClipMap.get(lastClip);
                     }
                     else {
                         // We're creating a "hold" clip here; exactly 1 second long, and inteprolates exactly on the duration of the clip

package/src/engine-components/ui/Button.ts

@@ -180,6 +180,8 @@ export class Button extends Behaviour implements IPointerEventHandler {
             this.onClick.invoke();
             args.use();
 
+            this.context.accessibility.focus(this);
+
             // debug clicks for WebXR
             if (debug) {
                 const pos = this.gameObject.worldPosition;
@@ -238,9 +240,19 @@ export class Button extends Behaviour implements IPointerEventHandler {
 
     onEnable() {
         super.onEnable();
+        this.context.accessibility.updateElement(this, {
+            role: "button",
+            label: this.gameObject.name + " button",
+            hidden: false
+        });
+    }
+    onDisable() {
+        super.onDisable();
+        this.context.accessibility.updateElement(this, { hidden: true })
     }
 
     onDestroy(): void {
+        this.context.accessibility.removeElement(this);
         if (this._isHovered) this.context.input.unsetCursor("pointer");
     }
 

package/src/engine-components/ui/Text.ts

@@ -102,6 +102,7 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
             this._text = val;
             this.feedText(this.text, this.supportRichText);
             this.markDirty();
+            this.context.accessibility.updateElement(this, { label: this.text });
         }
     }
 
@@ -238,6 +239,13 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
 
     onEnable(): void {
         super.onEnable();
+
+        this.context.accessibility.updateElement(this, {
+            role: "text",
+            label: this.text,
+            hidden: false
+        });
+
         this._didHandleTextRenderOnTop = false;
         if (this.uiObject) {
             // @ts-ignore
@@ -261,6 +269,11 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
     onDisable(): void {
         super.onDisable();
         this.canvas?.unregisterEventReceiver(this);
+        this.context.accessibility.updateElement(this, { hidden: true });
+    }
+    onDestroy(): void {
+        super.onDestroy();
+        this.context.accessibility.removeElement(this);
     }
 
     private getAlignment(opts: ThreeMeshUIEveryOptions): ThreeMeshUIEveryOptions {