@needle-tools/engine 3.3.0-alpha → 3.5.0-alpha

package/src/engine-components/export/usdz/USDZExporter.ts

@@ -1,15 +1,15 @@
 import { delay, getParam, isiOS, isMobileDevice, isSafari } from "../../../engine/engine_utils";
-import { Object3D, Color, Mesh, Matrix4 } from "three";
-import { USDZExporter as ThreeUSDZExporter } from "three/examples/jsm/exporters/USDZExporter";
+import { Object3D, Mesh, Matrix4 } from "three";
+import { USDZExporter as ThreeUSDZExporter } from "./ThreeUSDZExporter";
 import { AnimationExtension } from "./extensions/Animation"
 import { ensureQuicklookLinkIsCreated } from "./utils/quicklook";
 import { getFormattedDate } from "./utils/timeutils";
 import { registerAnimatorsImplictly } from "./utils/animationutils";
-import { IUSDZExporterExtension } from "./Extension";
+import { IUSDExporterExtension } from "./Extension";
 import { Behaviour, GameObject } from "../../Component";
 import { WebXR } from "../../webxr/WebXR"
 import { serializable } from "../../../engine/engine_serialization";
-import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../../engine/debug/debug";
+import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/debug";
 import { Context } from "../../../engine/engine_setup";
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot";
 import { hasProLicense } from "../../../engine/engine_license";
@@ -32,7 +32,7 @@ export class QuickLookOverlay {
 export class USDZExporter extends Behaviour {
 
     @serializable(Object3D)
-    objectToExport?: THREE.Object3D;
+    objectToExport?: Object3D;
 
     @serializable()
     autoExportAnimations: boolean = false;
@@ -43,13 +43,20 @@ export class USDZExporter extends Behaviour {
     @serializable(QuickLookOverlay)
     overlay?: QuickLookOverlay;
 
-    extensions: IUSDZExporterExtension[] = [];
+    // Currently not exposed to integrations - not fully tested. Set from code (e.g. image tracking)
+    @serializable()
+    anchoringType: "plane" | "image" | "face" | "none" = "plane";
+
+    // Currently not exposed to integrations - not fully tested. Set from code (e.g. image tracking)
+    @serializable()
+    planeAnchoringAlignment: "horizontal" | "vertical" | "any" = "horizontal";
+
+    extensions: IUSDExporterExtension[] = [];
 
     private link!: HTMLAnchorElement;
     private webxr?: WebXR;
     private webARSessionRoot: WebARSessionRoot | undefined;
 
-
     start() {
         if (debug) {
             console.log(this);
@@ -72,13 +79,11 @@ export class USDZExporter extends Behaviour {
             this.exportAsync();
         });
 
-        if (!this.objectToExport) this.objectToExport = this.gameObject;
-
-
-        if (isDevEnvironment() && (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)) {
-            showBalloonWarning("USDZ Exporter has nothing to export");
-            console.warn("USDZExporter has no objects to export assigned:", this)
-        }
+        // fall back to this object or to the scene if it's empty and doesn't have a mesh
+        if (!this.objectToExport)
+            this.objectToExport = this.gameObject;
+        if (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)
+            this.objectToExport = this.context.scene;
     }
 
 
@@ -135,13 +140,30 @@ export class USDZExporter extends Behaviour {
         this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
 
         let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
-        if (debug) name += "-" + getFormattedDate();
-        else if (!hasProLicense()) name = name + " - Made with Needle";
+        if (!hasProLicense()) name += "-MadeWithNeedle";
+        name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
 
         //@ts-ignore
         exporter.debug = debug;
+        
+        // sanitize anchoring types
+        if (this.anchoringType !== "plane" && this.anchoringType !== "none" && this.anchoringType !== "image" && this.anchoringType !== "face")
+            this.anchoringType = "plane";
+        if (this.planeAnchoringAlignment !== "horizontal" && this.planeAnchoringAlignment !== "vertical" && this.planeAnchoringAlignment !== "any")
+            this.planeAnchoringAlignment = "horizontal";
+
         //@ts-ignore
-        const arraybuffer = await exporter.parse(this.objectToExport, extensions);
+        const arraybuffer = await exporter.parse(this.objectToExport, {
+            ar: {
+                anchoring: {
+                    type: this.anchoringType,
+                }
+            },
+            planeAnchoring: {
+                alignment: this.planeAnchoringAlignment,
+            },
+            extensions: extensions
+        });
         const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
 
         this.dispatchEvent(new CustomEvent("after-export", { detail: eventArgs }))

package/src/engine-components/export/usdz/extensions/Animation.ts

@@ -1,17 +1,18 @@
 import { GameObject } from "../../../Component";
 import { getParam } from "../../../../engine/engine_utils";
-import { Object3D, Color, Matrix4, MeshStandardMaterial, Vector3, Quaternion, Interpolant } from "three";
-//@ts-ignore
-import { USDZObject, buildMatrix } from "three/examples/jsm/exporters/USDZExporter"
-import { IUSDZExporterExtension } from "../Extension";
+
+import { USDObject, buildMatrix } from "../ThreeUSDZExporter";
+import { IUSDExporterExtension } from "../Extension";
+
+import { Object3D, Matrix4, Vector3, Quaternion, Interpolant, AnimationClip, KeyframeTrack } from "three";
 
 const debug = getParam("debugusdzanimation");
 
 export interface UsdzAnimation {
-    createAnimation(ext: AnimationExtension, model: USDZObject, context);
+    createAnimation(ext: AnimationExtension, model: USDObject, context);
 }
 
-export type AnimationClipCollection = Array<{ root: Object3D, clips: Array<THREE.AnimationClip> }>;
+export type AnimationClipCollection = Array<{ root: Object3D, clips: Array<AnimationClip> }>;
 
 export class RegisteredAnimationInfo {
 
@@ -20,9 +21,9 @@ export class RegisteredAnimationInfo {
 
     private ext: AnimationExtension;
     private root: Object3D;
-    private clip: THREE.AnimationClip;
+    private clip: AnimationClip;
 
-    constructor(ext: AnimationExtension, root: THREE.Object3D, clip: THREE.AnimationClip) {
+    constructor(ext: AnimationExtension, root: Object3D, clip: AnimationClip) {
         this.ext = ext;
         this.root = root;
         this.clip = clip;
@@ -30,17 +31,17 @@ export class RegisteredAnimationInfo {
 }
 
 export class TransformData {
-    clip: THREE.AnimationClip;
-    pos?: THREE.KeyframeTrack;
-    rot?: THREE.KeyframeTrack;
-    scale?: THREE.KeyframeTrack;
+    clip: AnimationClip;
+    pos?: KeyframeTrack;
+    rot?: KeyframeTrack;
+    scale?: KeyframeTrack;
     get frameRate(): number { return 60; }
 
     private ext: AnimationExtension;
     private root: Object3D;
     private target: Object3D;
 
-    constructor(ext: AnimationExtension, root: Object3D, target: Object3D, clip: THREE.AnimationClip) {
+    constructor(ext: AnimationExtension, root: Object3D, target: Object3D, clip: AnimationClip) {
         this.ext = ext;
         this.root = root;
         this.target = target;
@@ -78,14 +79,14 @@ export class TransformData {
 
 declare type AnimationDict = Map<Object3D, Array<TransformData>>;
 
-export class AnimationExtension implements IUSDZExporterExtension {
+export class AnimationExtension implements IUSDExporterExtension {
 
     get extensionName(): string { return "animation" }
     private dict: AnimationDict = new Map();
     // private rootTargetMap: Map<Object3D, Object3D[]> = new Map();
     private rootTargetMap: Map<Object3D, Object3D[]> = new Map();
 
-    getStartTime01(root: Object3D, clip: THREE.AnimationClip) {
+    getStartTime01(root: Object3D, clip: AnimationClip) {
         const targets = this.rootTargetMap.get(root);
         if (!targets) return Infinity;
         let longestStartTime: number = -1;
@@ -108,7 +109,7 @@ export class AnimationExtension implements IUSDZExporterExtension {
         return longestStartTime;
     }
 
-    registerAnimation(root: Object3D, clip: THREE.AnimationClip): RegisteredAnimationInfo | null {
+    registerAnimation(root: Object3D, clip: AnimationClip): RegisteredAnimationInfo | null {
         if (!clip || !root) return null;
         if (!this.rootTargetMap.has(root)) this.rootTargetMap.set(root, []);
         // this.rootTargetMap.get(root)?.push(clip);
@@ -165,7 +166,7 @@ export class AnimationExtension implements IUSDZExporterExtension {
         }
     }
 
-    onExportObject(object, model: USDZObject, _context) {
+    onExportObject(object, model: USDObject, _context) {
 
         GameObject.foreachComponent(object, (comp) => {
             const c = comp as unknown as UsdzAnimation;
@@ -187,7 +188,7 @@ class SerializeAnimation {
 
     object: Object3D;
     dict: AnimationDict;
-    model: USDZObject;
+    model: USDObject | undefined = undefined;
 
     private callback?: Function;
 
@@ -196,7 +197,7 @@ class SerializeAnimation {
         this.dict = dict;
     }
 
-    registerCallback(model: USDZObject) {
+    registerCallback(model: USDObject) {
         if (this.model && this.callback) {
             this.model.removeEventListener("serialize", this.callback);
         }
@@ -209,6 +210,7 @@ class SerializeAnimation {
     }
 
     onSerialize(writer, _context) {
+        if (!this.model) return;
         if (debug)
             console.log("SERIALIZE", this.model.name, this.object.type);
         // do we have a track for this?
@@ -228,22 +230,24 @@ class SerializeAnimation {
         const rotation = new Quaternion();
         const scale = new Vector3(1, 1, 1);
 
-        // TODO doesn't support individual time arrays right now
-        // could use these in case we don't have time values that are identical
-        /*
-        const translationInterpolant = o.pos?.createInterpolant() as THREE.Interpolant;
-        const rotationInterpolant = o.rot?.createInterpolant() as THREE.Interpolant;
-        const scaleInterpolant = o.scale?.createInterpolant() as THREE.Interpolant;
-        */
-
         writer.appendLine("matrix4d xformOp:transform.timeSamples = {");
         writer.indent++;
 
         for (const transformData of arr) {
-            let timesArray = transformData.pos?.times;
-            if (!timesArray || transformData.rot && transformData.rot.times?.length > timesArray?.length) timesArray = transformData.rot?.times;
-            if (!timesArray || transformData.scale && transformData.scale.times?.length > timesArray?.length) timesArray = transformData.scale?.times;
-            if (!timesArray) {
+            let posTimesArray = transformData.pos?.times;
+            let rotTimesArray = transformData.rot?.times;
+            let scaleTimesArray = transformData.scale?.times;
+            
+            // timesArray is the sorted union of all time values
+            let timesArray: number[] = [];
+            if (posTimesArray) for (const t of posTimesArray) timesArray.push(t);
+            if (rotTimesArray) for (const t of rotTimesArray) timesArray.push(t);
+            if (scaleTimesArray) for (const t of scaleTimesArray) timesArray.push(t);
+            // sort
+            timesArray.sort((a, b) => a - b);
+            timesArray = [...new Set(timesArray)];
+
+            if (!timesArray || timesArray.length === 0) {
                 console.error("got an animated object but no time values??", object, transformData);
                 continue;
             }
@@ -276,8 +280,8 @@ class SerializeAnimation {
                     rotation.set(quat[0], quat[1], quat[2], quat[3]);
                 }
                 if (scaleInterpolant) {
-                    const scale = scaleInterpolant.evaluate(time);
-                    scale.set(scale[0], scale[1], scale[2]);
+                    const scaleVal = scaleInterpolant.evaluate(time);
+                    scale.set(scaleVal[0], scaleVal[1], scaleVal[2]);
                 }
 
                 composedTransform.compose(translation, rotation, scale);
@@ -290,17 +294,5 @@ class SerializeAnimation {
         }
         writer.indent--;
         writer.appendLine("}");
-
-        /*
-        let transform3 = new Matrix4();
-        transform3.compose(0.2,0,0);
-        const transform = buildMatrix(model.matrix);
-        const transform2 = buildMatrix(transform3.multiply(model.matrix));
-        
-        writer.appendLine(`matrix4d xformOp:transform.timeSamples = {
-            0: ${transform},
-            30: ${transform2}
-        }`);
-        */
     }
 }

package/src/engine-components/export/usdz/extensions/DocumentExtension.ts

@@ -0,0 +1,10 @@
+import { IUSDExporterExtension } from "../Extension";
+
+export class DocumentExtension implements IUSDExporterExtension {
+
+    get extensionName(): string {
+        return "DocumentExtension";
+    }
+
+    onAfterBuildDocument(_context: any) { }
+}

package/src/engine-components/export/usdz/extensions/USDZText.ts

@@ -0,0 +1,142 @@
+import { IBehaviorElement } from "../extensions/behavior/BehavioursBuilder";
+import { USDDocument, USDWriter } from "../ThreeUSDZExporter";
+
+
+export enum TextWrapMode {
+    singleLine = "singleLine",
+    hardBreaks = "hardBreaks",
+    flowing = "flowing",
+}
+
+export enum HorizontalAlignment {
+    left = "left",
+    center = "center",
+    right = "right",
+    justified = "justified"
+}
+
+export enum VerticalAlignment {
+    top = "top",
+    middle = "middle",
+    lowerMiddle = "lowerMiddle",
+    baseline = "baseline",
+    bottom = "bottom"
+}
+
+export class USDZText implements IBehaviorElement {
+
+    static global_id: number = 0;
+    static getId(): number {
+        return this.global_id++;
+    }
+
+    id: string;
+    content: string = "";
+    font?: string[] = [];
+    pointSize: number = 144;
+    width?: number;
+    height?: number;
+    depth?: number;
+    wrapMode?: TextWrapMode;
+    horizontalAlignment?: HorizontalAlignment;
+    verticalAlignment?: VerticalAlignment;
+
+    setDepth(depth: number): USDZText {
+        this.depth = depth;
+        return this;
+    }
+
+    setPointSize(pointSize: number): USDZText {
+        this.pointSize = pointSize;
+        return this;
+    }
+
+    setHorizontalAlignment(align: HorizontalAlignment) {
+        this.horizontalAlignment = align;
+        return this;
+    }
+
+    setVerticalAlignment(align: VerticalAlignment) {
+        this.verticalAlignment = align;
+        return this;
+    }
+
+    constructor(id: string) {
+        this.id = id;
+    }
+
+    writeTo(_document: USDDocument | undefined, writer: USDWriter) {
+
+
+        writer.beginBlock(`def Preliminary_Text "${this.id}"`);
+
+        if (this.content)
+            writer.appendLine(`string content = "${this.content}"`);
+
+        if (!this.font || this.font.length <= 0) {
+            this.font ||= [];
+            this.font?.push("sans-serif");
+        }
+        const str = this.font.map(s => `"${s}"`).join(", ");
+        writer.appendLine(`string[] font = [ ${str} ]`);
+
+        writer.appendLine(`double pointSize = ${this.pointSize}`);
+        if (typeof this.width === "number")
+            writer.appendLine(`double width = ${this.width}`);
+        if (typeof this.height === "number")
+            writer.appendLine(`double height = ${this.height}`);
+        if (typeof this.depth === "number")
+            writer.appendLine(`double depth = ${this.depth}`);
+        if (this.wrapMode)
+            writer.appendLine(`token wrapMode = "${this.wrapMode}"`);
+        if (this.horizontalAlignment)
+            writer.appendLine(`token horizontalAlignment = "${this.horizontalAlignment}"`);
+        if (this.verticalAlignment)
+            writer.appendLine(`token verticalAlignment = "${this.verticalAlignment}"`);
+
+        writer.closeBlock();
+
+    }
+
+}
+
+
+export class TextBuilder {
+    static singleLine(str: string, pointSize?: number, depth?: number): USDZText {
+
+        const text = new USDZText("text_" + USDZText.getId());
+        text.content = str;
+        if (pointSize)
+            text.pointSize = pointSize;
+        if (depth)
+            text.depth = depth;
+        return text;
+    }
+
+    static multiLine(str: string, width: number, height: number, horizontal: HorizontalAlignment, vertical: VerticalAlignment, wrapMode?: TextWrapMode) {
+        const text = new USDZText("text_" + USDZText.getId());
+        text.content = str;
+        text.width = width;
+        text.height = height;
+        text.horizontalAlignment = horizontal;
+        text.verticalAlignment = vertical;
+        if (wrapMode)
+            text.wrapMode = wrapMode;
+        return text;
+    }
+}
+
+
+export class TextExtension {
+    onExportObject(_object, model, _context) {
+        model.addEventListener("serialize", (writer, _context) => {
+            const text = TextBuilder.multiLine("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.",
+                1, 1, HorizontalAlignment.justified, VerticalAlignment.top);
+            text.pointSize = 300;
+            text.depth = .01;
+            text.writeTo(undefined, writer);
+        });
+    }
+}
+
+

package/src/engine-components/export/usdz/extensions/behavior/Actions.ts

@@ -0,0 +1,99 @@
+import { Object3D, Matrix4, Material, BufferGeometry } from "three";
+import { ActionBuilder, ActionModel } from "./BehavioursBuilder";
+import { USDObject, USDDocument } from "../../ThreeUSDZExporter";
+
+export abstract class DocumentAction {
+
+    get id(): string { return this.object.uuid; }
+
+    protected object: Object3D;
+    protected model?: USDObject;
+
+    constructor(obj: Object3D) {
+        this.object = obj;
+    }
+
+    apply(document: USDDocument) {
+        if (!this.model) {
+            this.model = document.findById(this.object.uuid);
+            if (!this.model) {
+                console.error("could not find model with id " + this.object.uuid);
+                return;
+            }
+        }
+        this.onApply(document);
+    }
+
+    protected abstract onApply(document: USDDocument);
+}
+
+export class VariantAction extends DocumentAction {
+    constructor(obj: Object3D, matrix?: Matrix4, material?: Material, geometry?: BufferGeometry) {
+        super(obj);
+        this.matrix = matrix;
+        this.material = material;
+        this.geometry = geometry;
+    }
+
+    private matrix: Matrix4 | undefined;
+    private material: Material | undefined;
+    private geometry: BufferGeometry | undefined;
+
+    protected onApply(_: USDDocument) {
+        const model = this.model;
+        if (!model) return;
+        if (!model.parent?.isDynamic) {
+            USDObject.createEmptyParent(model);
+        }
+        const clone = model.clone();
+        if (this.matrix) clone.matrix = this.matrix;
+        if (this.material) clone.material = this.material;
+        if (this.geometry) clone.geometry = this.geometry;
+        model.parent?.add(clone);
+    }
+
+    private _enableAction?: ActionModel;
+    private _disableAction?: ActionModel;
+
+    enable(): ActionModel {
+        if (this._enableAction) return this._enableAction;
+        this._enableAction = ActionBuilder.fadeAction(this.object, 0, true);;
+        return this._enableAction;
+    }
+
+    disable(): ActionModel {
+        if (this._disableAction) return this._disableAction;
+        this._disableAction = ActionBuilder.fadeAction(this.object, 0, false);;
+        return this._disableAction;
+    }
+}
+
+export class ActionCollection {
+
+    private actions: DocumentAction[];
+    private sortedActions?: { [key: string]: DocumentAction[] };
+
+    constructor(actions: DocumentAction[]) {
+        this.actions = [...actions]
+    }
+
+    // organize is called once when getting an action for the first time
+    // the sorted actions are baked then and adding new actions will not be added anymore
+    private organize() {
+        this.sortedActions = {};
+        for (const action of this.actions) {
+            const id = action.id;
+            if (!this.sortedActions[id]) {
+                this.sortedActions[id] = [];
+            }
+            this.sortedActions[id].push(action);
+        }
+    }
+
+    /** returns all document actions affecting the object passed in */
+    getActions(obj: Object3D): DocumentAction[] | null {
+        if (!this.sortedActions) this.organize();
+        return this.sortedActions![obj.uuid];
+    }
+
+}