@needle-tools/engine 3.5.1-alpha → 3.5.3-alpha

package/src/engine-components/export/usdz/ThreeUSDZExporter.ts

@@ -1,3 +1,5 @@
+import { Renderer } from '../../Renderer';
+import { GameObject } from '../../Component';
 import {
 	PlaneGeometry,
 	Texture,
@@ -19,6 +21,9 @@ import {
 	MeshStandardMaterial,
 	sRGBEncoding,
 	MeshPhysicalMaterial,
+	Object3D,
+	MeshBasicMaterial,
+	SkinnedMesh,
 } from 'three';
 import * as fflate from 'three/examples/jsm/libs/fflate.module.js';
 
@@ -48,7 +53,6 @@ class USDObject {
 	parent: USDObject | null;
 	children: Array<USDObject | null> = [];
 	_eventListeners: {};
-	mesh: any;
 
 	static createEmptyParent( object ) {
 
@@ -92,7 +96,7 @@ class USDObject {
 
 	clone() {
 
-		const clone = new USDObject( MathUtils.generateUUID(), this.name, this.matrix, this.mesh, this.material );
+		const clone = new USDObject( MathUtils.generateUUID(), this.name, this.matrix, this.geometry, this.material );
 		clone.isDynamic = this.isDynamic;
 		return clone;
 
@@ -383,9 +387,20 @@ class USDZExporterContext {
 
 }
 
+/**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_anchoring_type) */
+export type Anchoring = "plane" | "image" | "face" | "none"
+/**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_planeanchoring_alignment) */
+export type Alignment = "horizontal" | "vertical" | "any";
+
 class USDZExporterOptions {
-	ar: { anchoring: { type: string } } = { anchoring: { type: 'plane' } };
-	planeAnchoring: { alignment: string } = { alignment: 'horizontal' };
+	ar: {
+		anchoring: { type: Anchoring },
+		planeAnchoring: { alignment: Alignment },
+	} = { 
+		anchoring: { type: 'plane' },
+		planeAnchoring: { alignment: 'horizontal' }
+	};
+	quickLookCompatible: boolean = false;
 	extensions: any[] = [];
 }
 
@@ -426,7 +441,7 @@ class USDZExporter {
 
 		parseDocument( context );
 
-		invokeAll( context, 'onAfterSerialize' );
+		await invokeAll( context, 'onAfterSerialize' );
 
 		context.output += buildMaterials( materials, textures );
 
@@ -499,21 +514,33 @@ class USDZExporter {
 
 }
 
-function traverseVisible( object, parentModel, context ) {
+function traverseVisible( object: Object3D, parentModel: USDObject, context: USDZExporterContext ) {
 
 	if ( ! object.visible ) return;
-
+	
 	let model: USDObject | undefined = undefined;
-	const geometry = object.geometry;
-	const material = object.material;
+	let geometry: BufferGeometry | undefined = undefined;
+	let material: Material | Material[] | undefined = undefined;
+	
+	if (object instanceof Mesh) {
+		geometry = object.geometry;
+		material = object.material;
+	}
 
+	// TODO what should be do with disabled renderers?
+	// Here we just assume they're off, and don't export them
+	const renderer = GameObject.getComponent( object, Renderer )
+	if (renderer && !renderer.enabled) {
+		geometry = undefined;
+		material = undefined;
+	}
 
-	if ( object.isMesh && material && (material.isMeshStandardMaterial || material.isMeshBasicMaterial) && ! object.isSkinnedMesh ) {
+	if ( object instanceof Mesh && material && (material instanceof MeshStandardMaterial || material instanceof MeshBasicMaterial) && ! (object instanceof SkinnedMesh )) {
 
 		const name = getObjectId( object );
 		model = new USDObject( object.uuid, name, object.matrix, geometry, material );
 
-	} else if ( object.isCamera ) {
+	} else if ( object instanceof Camera ) {
 
 		const name = getObjectId( object );
 		model = new USDObject( object.uuid, name, object.matrix, undefined, undefined, object );
@@ -567,7 +594,7 @@ function traverseVisible( object, parentModel, context ) {
 
 }
 
-function parseDocument( context: USDZExporterContext ) {
+async function parseDocument( context: USDZExporterContext ) {
 
 	for ( const child of context.document.children ) {
 
@@ -594,7 +621,7 @@ function parseDocument( context: USDZExporterContext ) {
 
 	writer.appendLine( `token preliminary:anchoring:type = "${context.exporter.sceneAnchoringOptions.ar.anchoring.type}"` );
 	if (context.exporter.sceneAnchoringOptions.ar.anchoring.type === 'plane')
-		writer.appendLine( `token preliminary:planeAnchoring:alignment = "${context.exporter.sceneAnchoringOptions.planeAnchoring.alignment}"` );
+		writer.appendLine( `token preliminary:planeAnchoring:alignment = "${context.exporter.sceneAnchoringOptions.ar.planeAnchoring.alignment}"` );
 	// bit hacky as we don't have a callback here yet. Relies on the fact that the image is named identical in the ImageTracking extension.
 	if (context.exporter.sceneAnchoringOptions.ar.anchoring.type === 'image')
 		writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
@@ -656,14 +683,27 @@ function addResources( object, context: USDZExporterContext ) {
 
 }
 
-function invokeAll( context: USDZExporterContext, name: string, writer: USDWriter | null = null ) {
+async function invokeAll( context: USDZExporterContext, name: string, writer: USDWriter | null = null ) {
 
 	if ( context.extensions ) {
 
 		for ( const ext of context.extensions ) {
 
-			if ( typeof ext[ name ] === 'function' )
-				ext[ name ]( context, writer );
+			if ( !ext ) continue;
+
+			if ( typeof ext[ name ] === 'function' ) {
+
+				const method = ext[ name ];
+
+				const isAsync = method.constructor.name === "AsyncFunction";
+				
+				if ( isAsync ) {
+					await method.call( ext, context, writer );
+				} else {
+					method.call( ext, context, writer );
+				}
+
+			}
 
 		}
 
@@ -831,6 +871,14 @@ export function buildXform( model, writer, context ) {
 	const material = model.material;
 	const camera = model.camera;
 	const name = model.name;
+
+	// postprocess node
+	if ( model.onBeforeSerialize ) {
+
+		model.onBeforeSerialize( writer, context );
+
+	}
+
 	const transform = buildMatrix( matrix );
 
 	if ( matrix.determinant() < 0 ) {
@@ -839,11 +887,12 @@ export function buildXform( model, writer, context ) {
 
 	}
 
-	if ( geometry )
+	if ( geometry ) {
 		writer.beginBlock( `def Xform "${name}" (
-			prepend references = @./geometries/Geometry_${geometry.id}.usd@</Geometry>
-			prepend apiSchemas = ["MaterialBindingAPI"]
+prepend references = @./geometries/Geometry_${geometry.id}.usd@</Geometry>
+prepend apiSchemas = ["MaterialBindingAPI"]
 		)` );
+	}
 	else if ( camera )
 		writer.beginBlock( `def Camera "${name}"` );
 	else
@@ -1076,7 +1125,7 @@ function buildMaterial( material: MeshStandardMaterial, textures ) {
 	const pad = '            ';
 	const inputs: Array<string> = [];
 	const samplers: Array<string> = [];
-	const exportForQuickLook = true;
+	const exportForQuickLook = false;
 
 	function buildTexture( texture, mapType, color: Color | undefined = undefined, opacity: number | undefined = undefined ) {
 
@@ -1088,6 +1137,11 @@ function buildMaterial( material: MeshStandardMaterial, textures ) {
 
 		const repeat = texture.repeat.clone();
 		const offset = texture.offset.clone();
+		const rotation = texture.rotation;
+
+		// rotation is around the wrong point. after rotation we need to shift offset again so that we're rotating around the right spot
+		let xRotationOffset = Math.sin(rotation);
+		let yRotationOffset = Math.cos(rotation);
 
 		// texture coordinates start in the opposite corner, need to correct
 		offset.y = 1 - offset.y - repeat.y;
@@ -1096,15 +1150,28 @@ function buildMaterial( material: MeshStandardMaterial, textures ) {
 		// Apple Feedback: 	FB10036297 and FB11442287
 		if ( exportForQuickLook ) {
 
+			// This is NOT correct yet in QuickLook, but comes close for a range of models.
+			// It becomes more incorrect the bigger the offset is
+
 			offset.x = offset.x / repeat.x;
 			offset.y = offset.y / repeat.y;
 
+			offset.x += xRotationOffset / repeat.x;
+			offset.y += yRotationOffset - 1;
+		}
+
+		else {
+
+			// results match glTF results exactly. verified correct in usdview.
+			offset.x += xRotationOffset * repeat.x;
+			offset.y += (1 - yRotationOffset) * repeat.y;
+
 		}
 
 		textures[ id ] = texture;
 		const uvReader = mapType == 'occlusion' ? 'uvReader_st2' : 'uvReader_st';
 
-		const needsTextureTransform = ( repeat.x != 1 || repeat.y != 1 || offset.x != 0 || offset.y != 0 );
+		const needsTextureTransform = ( repeat.x != 1 || repeat.y != 1 || offset.x != 0 || offset.y != 0 || rotation != 0 );
 		const textureTransformInput = `</Materials/Material_${material.id}/${uvReader}.outputs:result>`;
 		const textureTransformOutput = `</Materials/Material_${material.id}/Transform2d_${mapType}.outputs:result>`;
 
@@ -1123,6 +1190,7 @@ function buildMaterial( material: MeshStandardMaterial, textures ) {
             float2 inputs:in.connect = ${textureTransformInput}
             float2 inputs:scale = ${buildVector2( repeat )}
             float2 inputs:translation = ${buildVector2( offset )}
+			float inputs:rotation = ${(rotation / Math.PI * 180).toFixed( PRECISION )}
             float2 outputs:result
         }
 		` : '' }

package/src/engine-components/export/usdz/USDZExporter.ts

@@ -13,6 +13,8 @@ import { showBalloonMessage, showBalloonWarning } from "../../../engine/debug/de
 import { Context } from "../../../engine/engine_setup";
 import { WebARSessionRoot } from "../../webxr/WebARSessionRoot";
 import { hasProLicense } from "../../../engine/engine_license";
+import { BehaviorExtension } from "./extensions/behavior/Behaviour";
+import { AudioExtension } from "./extensions/behavior/AudioExtension";
 
 const debug = getParam("debugusdz");
 
@@ -40,6 +42,9 @@ export class USDZExporter extends Behaviour {
     @serializable()
     exportFileName?: string;
 
+    @serializable(URL)
+    customUsdzFile?: string;
+
     @serializable(QuickLookOverlay)
     overlay?: QuickLookOverlay;
 
@@ -51,6 +56,9 @@ export class USDZExporter extends Behaviour {
     @serializable()
     planeAnchoringAlignment: "horizontal" | "vertical" | "any" = "horizontal";
 
+    @serializable()
+    interactive: boolean = true;
+
     extensions: IUSDExporterExtension[] = [];
 
     private link!: HTMLAnchorElement;
@@ -84,9 +92,12 @@ export class USDZExporter extends Behaviour {
             this.objectToExport = this.gameObject;
         if (!this.objectToExport?.children?.length && !(this.objectToExport as Mesh)?.isMesh)
             this.objectToExport = this.context.scene;
-    }
-
 
+        if (this.interactive) {
+            this.extensions.push(new BehaviorExtension());
+            this.extensions.push(new AudioExtension());
+        }
+    }
 
     onEnable() {
         const ios = isiOS()
@@ -114,6 +125,35 @@ export class USDZExporter extends Behaviour {
     }
 
     async exportAsync() {
+        
+        let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
+        if (!hasProLicense()) name += "-MadeWithNeedle";
+        name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
+
+        // ability to specify a custom USDZ file to be used instead of a dynamic one
+        if (this.customUsdzFile) {
+
+            // see https://developer.apple.com/documentation/arkit/adding_an_apple_pay_button_or_a_custom_action_in_ar_quick_look
+            const overlay = this.buildQuicklookOverlay();
+            if(debug) console.log(overlay);
+            const callToAction = overlay.callToAction ? encodeURIComponent(overlay.callToAction) : "";
+            const checkoutTitle = overlay.checkoutTitle ? encodeURIComponent(overlay.checkoutTitle) : "";
+            const checkoutSubtitle = overlay.checkoutSubtitle ? encodeURIComponent(overlay.checkoutSubtitle) : "";
+            this.link.href = this.customUsdzFile + `#callToAction=${callToAction}&checkoutTitle=${checkoutTitle}&checkoutSubtitle=${checkoutSubtitle}&callToActionURL=${overlay.callToActionURL}`;
+
+            console.log(this.link.href)
+
+            if (!this.lastCallback) {
+                this.lastCallback = this.quicklookCallback.bind(this)
+                this.link.addEventListener('message', this.lastCallback);
+            }
+
+            // open quicklook
+            this.link.download = name + ".usdz";
+            this.link.click();
+            return;
+        }
+
         if (!this.objectToExport) return;
 
         // make sure we apply the AR scale
@@ -139,10 +179,6 @@ export class USDZExporter extends Behaviour {
         const eventArgs = { self: this, exporter: exporter, extensions: extensions, object: this.objectToExport };
         this.dispatchEvent(new CustomEvent("before-export", { detail: eventArgs }))
 
-        let name = this.exportFileName ?? this.objectToExport?.name ?? this.name;
-        if (!hasProLicense()) name += "-MadeWithNeedle";
-        name += "-" + getFormattedDate(); // seems iOS caches the file in some cases, this ensures we always have a fresh file
-
         //@ts-ignore
         exporter.debug = debug;
         
@@ -157,12 +193,13 @@ export class USDZExporter extends Behaviour {
             ar: {
                 anchoring: {
                     type: this.anchoringType,
-                }
-            },
-            planeAnchoring: {
-                alignment: this.planeAnchoringAlignment,
+                },
+                planeAnchoring: {
+                    alignment: this.planeAnchoringAlignment,
+                },
             },
-            extensions: extensions
+            extensions: extensions,
+            quickLookCompatible: true,
         });
         const blob = new Blob([arraybuffer], { type: 'application/octet-stream' });
 

package/src/engine-components/export/usdz/extensions/behavior/AudioExtension.ts

@@ -0,0 +1,63 @@
+import { GameObject } from "../../../../Component";
+import { IUSDExporterExtension } from "../../Extension";
+import { USDObject, USDWriter, USDZExporterContext } from "../../ThreeUSDZExporter";
+import { Object3D } from "three";
+import { AudioSource } from "../../../../AudioSource";
+
+export class AudioExtension implements IUSDExporterExtension {
+
+    get extensionName(): string {
+        return "Audio";
+    }
+
+    private files: string[] = [];
+
+    onExportObject?(object: Object3D, model : USDObject, _context: USDZExporterContext) {
+        // check if this object has an audio source, add the relevant schema in that case.
+        const audioSources = GameObject.getComponents(object, AudioSource);
+        if (audioSources.length) {
+            for (const audioSource of audioSources) {
+
+                // do nothing if this audio source is not set to play on awake - 
+                // should be controlled via PlayAudioOnClick instead then.
+                if (!audioSource.playOnAwake)
+                    continue;
+
+                const clipName = audioSource.clip.split("/").pop();
+
+                if (!this.files.includes(audioSource.clip)) {
+                    this.files.push(audioSource.clip);
+                }
+                
+                model.addEventListener('serialize', (writer: USDWriter, _context: USDZExporterContext) => {
+                    writer.appendLine();
+                    writer.beginBlock(`def SpatialAudio "${model.name}"`);
+                    writer.appendLine(`uniform asset filePath = @audio/${clipName}@`);
+                    writer.appendLine(`uniform token auralMode = "${ audioSource.spatialBlend > 0 ? "spatial" : "nonSpatial" }"`);
+                    // theoretically we could do timeline-like audio sequencing with this.
+                    writer.appendLine(`uniform token playbackMode = "${audioSource.loop ?  "loopFromStage" : "onceFromStart" }"`);
+                    writer.appendLine(`uniform float gain = ${audioSource.volume}`);
+                    writer.closeBlock();
+                });
+            }
+        }
+    }
+
+    async onAfterSerialize(context: USDZExporterContext) {
+        console.warn("onAfterSerialize", this);
+        // write the files to the context.
+        for (const file of this.files) {
+
+            const clipName = file.split("/").pop();
+
+            // convert file (which is a path) to a blob.
+            const audio = await fetch(file);
+            const audioBlob = await audio.blob();
+            const arrayBuffer = await audioBlob.arrayBuffer();
+
+            const audioData: Uint8Array = new Uint8Array(arrayBuffer)
+
+            context.files["audio/" + clipName] = audioData;
+        }
+    }
+}

package/src/engine-components/export/usdz/extensions/behavior/Behaviour.ts

@@ -1,23 +1,14 @@
-import { Behaviour, GameObject } from "../../../../Component";
-import { USDZExporter } from "../../USDZExporter";
+import { GameObject } from "../../../../Component";
+import { IContext } from "../../../../../engine/engine_types";
 import { IUSDExporterExtension } from "../../Extension";
 import { USDObject, USDWriter } from "../../ThreeUSDZExporter";
 import { BehaviorModel } from "./BehavioursBuilder";
-import { IContext } from "../../../../../engine/engine_types";
 
 export interface UsdzBehaviour {
     createBehaviours?(ext: BehaviorExtension, model: USDObject, context: IContext): void;
     beforeCreateDocument?(ext: BehaviorExtension, context: IContext): void;
     afterCreateDocument?(ext: BehaviorExtension, context: IContext): void;
-}
-
-export class USDZBehaviours extends Behaviour {
-    start() {
-        const exporter = GameObject.findObjectOfType(USDZExporter);
-        if (exporter) {
-            exporter.extensions.push(new BehaviorExtension());
-        }
-    }
+    afterSerialize?(ext: BehaviorExtension, context: IContext): void;
 }
 
 export class BehaviorExtension implements IUSDExporterExtension {
@@ -33,6 +24,7 @@ export class BehaviorExtension implements IUSDExporterExtension {
     }
 
     behaviourComponents: Array<UsdzBehaviour> = [];
+    private behaviourComponentsCopy: Array<UsdzBehaviour> = [];
 
 
     onBeforeBuildDocument(context) {
@@ -40,8 +32,8 @@ export class BehaviorExtension implements IUSDExporterExtension {
             GameObject.foreachComponent(e, (comp) => {
                 const c = comp as unknown as UsdzBehaviour;
                 if (
-                    typeof c.createBehaviours === "function" || 
-                    typeof c.beforeCreateDocument === "function" || 
+                    typeof c.createBehaviours === "function" ||
+                    typeof c.beforeCreateDocument === "function" ||
                     typeof c.afterCreateDocument === "function"
                 ) {
                     this.behaviourComponents.push(c);
@@ -52,7 +44,6 @@ export class BehaviorExtension implements IUSDExporterExtension {
     }
 
     onExportObject(_object, model: USDObject, context) {
-
         for (const beh of this.behaviourComponents) {
             beh.createBehaviours?.call(beh, this, model, context);
         }
@@ -63,10 +54,11 @@ export class BehaviorExtension implements IUSDExporterExtension {
             if (typeof beh.afterCreateDocument === "function")
                 beh.afterCreateDocument(this, context);
         }
+        this.behaviourComponentsCopy = this.behaviourComponents.slice();
         this.behaviourComponents.length = 0;
     }
 
-    onAfterHierarchy(context, writer : USDWriter) {
+    onAfterHierarchy(context, writer: USDWriter) {
         if (this.behaviours?.length) {
 
             // this.combineBehavioursWithSameTapActions();
@@ -80,6 +72,26 @@ export class BehaviorExtension implements IUSDExporterExtension {
         }
     }
 
+    async onAfterSerialize(context) {
+        console.log("onAfterSerialize", this.behaviourComponentsCopy)
+        for (const beh of this.behaviourComponentsCopy) {
+            
+            console.log("behaviour", beh)
+            if (typeof beh.afterSerialize === "function") {
+
+                console.log("beh has afterSerialize", beh)
+
+                const isAsync = beh.afterSerialize.constructor.name === "AsyncFunction";
+				
+				if ( isAsync ) {
+					await beh.afterSerialize(this, context);
+				} else {
+					beh.afterSerialize(this, context);
+				}
+            }
+        }
+    }
+
     // combine behaviours that have tap triggers on the same object
     // private combineBehavioursWithSameTapActions() {
     //     // TODO: if behaviours have different settings (e.g. one is exclusive and another one is not) this wont work - we need more logic for that