@needle-tools/engine 4.11.0-next.8bfb2f5 → 4.11.0-next.c7baa24

package/src/engine-components/SeeThrough.ts

@@ -1,11 +1,14 @@
-import { Material, Object3D, Vector3 } from "three";
+import { Material, Object3D, Object3DEventMap, Vector3 } from "three";
 
 import { Gizmos } from "../engine/engine_gizmos.js";
 import { Mathf } from "../engine/engine_math.js";
 import { serializable } from "../engine/engine_serialization_decorator.js";
 import { getTempVector } from "../engine/engine_three_utils.js";
 import { getParam } from "../engine/engine_utils.js";
+import { USDObject, USDZExporterContext } from "./api.js";
 import { Behaviour } from "./Component.js";
+import { IUSDExporterExtension } from "./export/usdz/Extension.js";
+import { USDZExporter } from "./export/usdz/USDZExporter.js";
 import { Renderer } from "./Renderer.js";
 
 const debugSeeThrough = getParam("debugseethrough");
@@ -102,6 +105,7 @@ export class SeeThrough extends Behaviour {
     onEnable() {
         this._needsUpdate = true;
         this._renderer = null;
+        SeeThroughUsdzExporterPlugin.components.push(this);
     }
 
     /**  @internal */
@@ -118,6 +122,9 @@ export class SeeThrough extends Behaviour {
             this.rendererMaterials.delete(r);
             this.rendererMaterialsOriginal.delete(r);
         });
+
+        const index = SeeThroughUsdzExporterPlugin.components.indexOf(this);
+        if (index !== -1) SeeThroughUsdzExporterPlugin.components.splice(index, 1);
     }
 
     /**
@@ -253,4 +260,37 @@ export class SeeThrough extends Behaviour {
         });
     }
 
-}
+}
+
+
+;
+class SeeThroughUsdzExporterPlugin implements IUSDExporterExtension {
+
+    static readonly components: SeeThrough[] = [];
+
+    get extensionName() {
+        return "SeeThrough";
+    }
+
+    // onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, context: USDZExporterContext) {
+    //     const component = SeeThroughUsdzExporterPlugin.components.find(c => c.gameObject === object);
+    //     if(!component) return;
+    //     console.log("OH MY GOD SEE THROUGH USDZ EXPORTER", component, model);
+
+    //     model.materialName = "AlphaHashMaterialInstance"; // we could make this unique per object if needed
+
+    //     model.addEventListener("serialize", (writer, context) => {
+    //         writer.appendLine(`# SeeThrough component on ${object.name}`);
+    //     });
+    // }
+
+}
+
+const seeThroughUsdzExporterPlugin = new SeeThroughUsdzExporterPlugin();
+
+USDZExporter.beforeExport.addEventListener(args => {
+    if (SeeThroughUsdzExporterPlugin.components.length === 0) return;
+    if (args.exporter.extensions.includes(seeThroughUsdzExporterPlugin) === false) {
+        args.exporter.extensions.push(seeThroughUsdzExporterPlugin);
+    }
+});

package/src/engine-components/export/usdz/ThreeUSDZExporter.ts

@@ -112,6 +112,10 @@ const PositionIdentity = new Vector3();
 const QuaternionIdentity = new Quaternion();
 const ScaleIdentity = new Vector3(1,1,1);
 
+// #region USDObject
+
+type USDObjectEventType = "serialize" & ({} & string);
+
 class USDObject {
 
 	static USDObject_export_id = 0;
@@ -153,12 +157,13 @@ class USDObject {
 	private set isDynamic( value ) { this._isDynamic = value; }
 	geometry: BufferGeometry | null;
 	material: MeshStandardMaterial | MeshBasicMaterial | Material | MeshPhysicalNodeMaterial | null;
+	// usdMaterial?: USDMaterial;
 	camera: PerspectiveCamera | OrthographicCamera | null;
 	parent: USDObject | null;
 	skinnedMesh: SkinnedMesh | null;
 	children: Array<USDObject | null> = [];
 	animations: AnimationClip[] | null;
-	_eventListeners: {};
+	_eventListeners: Record<USDObjectEventType, Function[]>;
 
 	// these are for tracking which xformops are needed
 	needsTranslate: boolean = false;
@@ -201,7 +206,7 @@ class USDObject {
 		this.camera = camera;
 		this.parent = null;
 		this.children = [];
-		this._eventListeners = {};
+		this._eventListeners = {} as Record<USDObjectEventType, Function[]>;
 		this._isDynamic = false;
 		this.skinnedMesh = skinnedMesh;
 		this.animations = animations;
@@ -287,7 +292,7 @@ class USDObject {
 
 	}
 
-	addEventListener( evt, listener: ( writer: USDWriter, context: USDZExporterContext ) => void ) {
+	addEventListener( evt : USDObjectEventType, listener: ( writer: USDWriter, context: USDZExporterContext ) => void ) {
 
 		if ( ! this._eventListeners[ evt ] ) this._eventListeners[ evt ] = [];
 		this._eventListeners[ evt ].push( listener );
@@ -316,6 +321,67 @@ class USDObject {
 }
 
 
+// #region USDMaterial
+
+// class MaterialInput {
+// 	name: string;
+// }
+
+class USDMaterial {
+
+	static USDMaterial_id = 0;
+
+	readonly material: Material;
+	readonly id: number;
+
+	name: string;
+
+	isOverride: boolean = false;
+	isInstanceable: boolean = false;
+
+	constructor( material: Material ) {
+		this.material = material;
+		this.id = USDMaterial.USDMaterial_id ++;
+		this.name = makeNameSafe( material.name || 'Material_' + this.id );
+	}
+
+	readonly inputs: Record<string, string | number | boolean | number[] | undefined> = {};
+
+	// addInput( name: string, value: "" ) {
+
+
+
+	// }
+
+
+	serialize( writer: USDWriter, _context: USDZExporterContext ) {
+
+		const name = this.name;
+
+		writer.appendLine( `def Material "${this.name}" ${name ?`( displayName = "${makeDisplayNameSafe(name)}" )` : ''}` );
+		writer.beginBlock();
+			
+
+		writer.closeBlock();
+		
+		// def Material "${materialName}" ${material.name ?`(
+		// 	displayName = "${material.name}"
+		// )` : ''}
+		// {
+		// 	token outputs:mtlx:surface.connect = ${materialRoot}/${materialName}/Occlusion.outputs:out>
+
+		// 	def Shader "Occlusion"
+		// 	{
+		// 		uniform token info:id = "${mode}"
+		// 		token outputs:out
+		// 	}
+		// }`;
+	
+	}
+}
+
+// #region USDDocument
+
 class USDDocument extends USDObject {
 
 	stageLength: number;
@@ -448,6 +514,7 @@ ${comments}
 
 }
 
+
 const newLine = '\n';
 const materialRoot = '</StageRoot/Materials';
 
@@ -540,6 +607,8 @@ class USDWriter {
 
 declare type TextureMap = {[name: string]: {texture: Texture, scale?: Vector4}};
 
+// #region USDZExporterContext
+
 class USDZExporterContext {
 	root?: Object3D;
 	exporter: USDZExporter;
@@ -576,6 +645,10 @@ class USDZExporterContext {
 
 	}
 
+	makeNameSafe( str ) {
+		return makeNameSafe( str );
+	}
+
 }
 
 /**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_anchoring_type) */
@@ -597,6 +670,8 @@ class USDZExporterOptions {
 	exportInvisible: boolean = false;
 }
 
+// #region USDZExporter
+
 class USDZExporter {
 	debug: boolean;
 	pruneUnusedNodes: boolean;
@@ -698,13 +773,7 @@ class USDZExporter {
 		}
 
 		Progress.report('export-usdz', { message: "Parsing document", autoStep: 10 });
-		await parseDocument( context, () => {
-			// injected after stageRoot.
-			// TODO property use context/writer instead of string concat
-			Progress.report('export-usdz', "Building materials");
-			const result = buildMaterials( materials, textures, options.quickLookCompatible );
-			return result;
-		} );
+		await parseDocument( context, options );
 
 		Progress.report("export-usdz", "Invoking onAfterSerialize");
 		await invokeAll( context, 'onAfterSerialize' );
@@ -726,7 +795,7 @@ class USDZExporter {
 		const final = header + '\n' + context.output;
 
 		// full output file
-		if ( this.debug ) console.log( final );
+		if ( this.debug ) console.debug( final );
 
 		files[ modelFileName ] = fflate.strToU8( final );
 		context.output = '';
@@ -822,6 +891,9 @@ class USDZExporter {
 
 }
 
+// #endregion
+
+// #region traverse 
 function traverse( object: Object3D, parentModel: USDObject, context: USDZExporterContext, keepObject?: (object: Object3D) => boolean ) {
 
 	if (!context.exportInvisible && !object.visible) return;
@@ -919,6 +991,8 @@ function traverse( object: Object3D, parentModel: USDObject, context: USDZExport
 
 }
 
+// #endregion
+
 function logUsdHierarchy( object: USDObject, prefix: string, ...extraLogObjects: any[] ) { 
 
 	const item = {};
@@ -1020,7 +1094,8 @@ function prune ( object: USDObject, options : {
 	return canBePruned;
 }
 
-async function parseDocument( context: USDZExporterContext, afterStageRoot: () => string ) {
+// #region parseDocument
+async function parseDocument( context: USDZExporterContext, options: USDZExporterOptions ) {
 
 	Progress.start("export-usdz-resources", "export-usdz");
 	const resources: Array<() => void> = [];
@@ -1086,14 +1161,21 @@ async function parseDocument( context: USDZExporterContext, afterStageRoot: () =
 
 	writer.closeBlock();
 	writer.closeBlock();
-	writer.appendLine(afterStageRoot());
+	
+	// TODO property use context/writer instead of string concat
+	Progress.report('export-usdz', "Building materials");
+	const result = buildMaterials( context.materials, context.textures, options.quickLookCompatible );
+	writer.appendLine(result);
+
 	writer.closeBlock();
 
 	Progress.report("export-usdz", "write to string")
 	context.output += writer.toString();
 
 }
+// #endregion
 
+// #region addResources
 function addResources( object: USDObject | null, context: USDZExporterContext, resources: Array<() => void>) {
 
 	if ( !object ) return;
@@ -1132,7 +1214,7 @@ function addResources( object: USDObject | null, context: USDZExporterContext, r
 
 	if ( material ) {
 		
-		if ( ! ( material.uuid in context.materials ) ) {
+		if ( context.materials.get( material.uuid ) === undefined ) {
 
 			context.materials[ material.uuid ] = material;
 
@@ -1169,6 +1251,10 @@ async function invokeAll( context: USDZExporterContext, name: string, writer: US
 	}
 
 }
+
+// #endregion
+
+// #region GPU utils
 let _renderer: WebGLRenderer | null = null;
 let renderTarget: WebGLRenderTarget | null = null;
 let fullscreenQuadGeometry: PlaneGeometry | null;
@@ -1431,7 +1517,9 @@ function getPathToSkeleton(bone: Object3D, assumedRoot: Object3D) {
 	return path;
 }
 
-// Xform
+// #endregion
+
+// #region XForm
 
 export function buildXform( model: USDObject | null, writer: USDWriter, context: USDZExporterContext ) {
 
@@ -1635,7 +1723,7 @@ function buildMatrixRow( array, offset ) {
 
 }
 
-// Mesh
+// #region Mesh
 
 function buildMeshObject( geometry: BufferGeometry, bonesArray: Bone[] = [], quickLookCompatible: boolean = true) {
 
@@ -1948,7 +2036,9 @@ function buildVector2Array( attribute: BufferAttribute | InterleavedBufferAttrib
 
 }
 
-// Materials
+// #endregion
+
+// #region Materials
 
 function buildMaterials( materials: Map<string, Material>, textures: TextureMap, quickLookCompatible = false ) {
 
@@ -2103,10 +2193,14 @@ function buildTexture( texture: Texture, mapType: MapType, quickLookCompatible:
 		}`;
 }
 
+
 function buildMaterial( material: MeshBasicMaterial, textures: TextureMap, quickLookCompatible = false ) {
 
 	const materialName = getMaterialName(material);
 
+	// TODO: refactor to USDMaterial class
+	// const usdMaterial = new USDMaterial(material);
+
 	// Special case: occluder material
 	// Supported on iOS 18+ and visionOS 1+
 	const isShadowCatcherMaterial = 
@@ -2330,6 +2424,7 @@ function buildMaterial( material: MeshBasicMaterial, textures: TextureMap, quick
 		}
 	}
 
+
 	return `
 
 		def Material "${materialName}" ${material.name ?`(
@@ -2368,6 +2463,8 @@ ${samplers.join( '\n' )}` : ''}
 		}`;
 }
 
+// #endregion
+
 function buildColor( color ) {
 
 	return `(${color.r}, ${color.g}, ${color.b})`;
@@ -2407,6 +2504,9 @@ const formatsWithAlphaChannel = [
 	36492, // RGBA_BPTC_Format
 ];
 
+
+// #region exports
+
 export { 
 	buildMatrix,
 	decompressGpuTexture,

package/src/engine-components/ui/Text.ts

@@ -284,7 +284,7 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
         return opts;
     }
 
-    private feedText(text: string, richText: boolean) : void {
+    private feedText(text: string, richText: boolean): void {
         // if (!text || text.length <= 0) return;
         // if (!text ) return;
         if (debug) console.log("feedText", this.uiObject, text, richText);
@@ -511,6 +511,15 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
     }
 
     private getFamilyNameWithCorrectSuffix(familyName: string, style: FontStyle): string {
+
+
+        // the URL decorator resolves the URL to absolute URLs - we need to remove the domain part since we're only interested in the path
+        if (familyName.startsWith("https:") || familyName.startsWith("http:")) {
+            const url = new URL(familyName);
+            familyName = url.pathname;
+        }
+
+
         // we can only change the style for the family if the name has a suffix (e.g. Arial-Bold)
         const styleSeparator = familyName.lastIndexOf('-');
         if (styleSeparator < 0) return familyName;
@@ -532,7 +541,7 @@ export class Text extends Graphic implements IHasAlphaFactor, ICanvasEventReceiv
             fontBaseName = fontBaseName.substring(pathSeparatorIndex + 1);
         }
         const isUpperCase = fontBaseName[0] === fontBaseName[0].toUpperCase();
-        const fontNameWithoutSuffix = familyName.substring(0, styleSeparator);
+        const fontNameWithoutSuffix = familyName.substring(0, styleSeparator > pathSeparatorIndex ? styleSeparator : familyName.length);
         if (debug) console.log("Select font: ", familyName, FontStyle[style], fontBaseName, isUpperCase, fontNameWithoutSuffix);
 
         switch (style) {

package/src/engine-components/utils/LookAt.ts

@@ -11,6 +11,9 @@ import { Behaviour } from "../Component.js";
 /**
  * LookAt behaviour makes the object look at a target object or the camera.  
  * It can also invert the forward direction and keep the up direction.
+ * 
+ * @category Interactivity, Everywhere Actions
+ * @group Components
  */
 export class LookAt extends Behaviour implements UsdzBehaviour {
 

package/src/engine-components/utils/OpenURL.ts

@@ -15,8 +15,9 @@ export enum OpenURLMode {
     NewWindow = 2
 }
 
-/**
- * OpenURL behaviour opens a URL in a new tab or window.
+/** 
+ * OpenURL behaviour opens a URL in a new tab or window when the object (or any if it's children) is clicked.
+ * 
  * @category Interactivity
  * @group Components
  */

package/src/engine-components/web/ScrollFollow.ts

@@ -473,7 +473,7 @@ function tryGetElementsForSelector(index: number): Element | null {
 
     if (!needsScrollMarkerRefresh) {
         const element = needleScrollMarkerCache[index] || null;
-        if (element) return element;
+        return element;
     }
     needsScrollMarkerRefresh = false;
     needleScrollMarkerCache.length = 0;

package/src/engine-components/webxr/WebARCameraBackground.ts

@@ -24,6 +24,8 @@ const debug = getParam("debugarcamera");
 /**
  * WebARCameraBackground is a component that allows to display the camera feed as a background in an AR session to more easily blend the real world with the virtual world or applying effects to the camera feed.
  * 
+ * - Example: https://samples.needle.tools/ar-camera-background
+ * 
  * @category XR
  * @group Components
  */

package/src/engine-components/webxr/WebARSessionRoot.ts

@@ -20,7 +20,7 @@ const invertForwardMatrix = new Matrix4().makeRotationY(Math.PI);
 
 /**
  * The WebARSessionRoot is the root object for a WebAR session and used to place the scene in AR.  
- * It is also responsible for scaling the user in AR. 
+ * It is also responsible for scaling the user in AR and to define the center of the AR scene. If not present in the scene it will be created automatically by the WebXR component when entering an AR session.
  * 
  * @example 
  * ```ts

package/src/engine-components/webxr/WebXR.ts

@@ -27,6 +27,8 @@ const debugQuicklook = getParam("debugusdz");
  * 
  * The WebXR component is a simple to use wrapper around the {@link NeedleXRSession} API and adds some additional features like creating buttons for AR, VR, enabling default movement behaviour ({@link XRControllerMovement}) and controller rendering ({@link XRControllerModel}), as well as handling AR placement and Quicklook USDZ export.
  * 
+ * - Example: https://samples.needle.tools/collaborative-sandbox
+ * 
  * @example Enable VR and AR support using code
  * ```ts
  * import { onStart, WebXR } from "@needle-tools/engine";

package/src/engine-components/webxr/WebXRImageTracking.ts

@@ -290,6 +290,16 @@ class ImageTrackingExtension implements IUSDExporterExtension {
 
 
 /**
+ * Add this component to a object to enable image tracking in WebXR sessions.    
+ * 
+ * You need to add at least one {@link WebXRImageTrackingModel} to the `trackedImages` array to define which images to track and which objects to place on top of them.
+ * 
+ * **NOTE:** For Android devices, image tracking currently requires the user to enable the `chrome://flags/#webxr-incubations` flag in Chrome.  
+ * 
+ * **NOTE:** For iOS only one image can be tracked at a time. If you have multiple images in the `trackedImages` array, only the first one will be tracked. You can use the {@link setPrimaryImage} method to change which image is tracked before entering the XR session.  
+ * 
+ * - Example: https://samples.needle.tools/image-tracking
+ * 
  * @category XR
  * @group Components
  */
@@ -313,7 +323,14 @@ export class WebXRImageTracking extends Behaviour {
 
     /**
      * Add an image to track. If the image is already in the trackedImages array it won't be added again.  
-     * Note: that adding images at runtime *while* in AR is not supported.
+     * Note: that adding images at runtime *while* in AR is not supported.  
+     * 
+     * @example
+     * ```ts
+     * const imageTracking = GameObject.getComponent(myGameObject, WebXRImageTracking);
+     * const image = new WebXRImageTrackingModel({ url: "https://example.com/my-marker.png", widthInMeters: 0.2, object: myObject });
+     * imageTracking.addImage(image, true); // add and set as primary image
+     * ```
      */
     addImage(image: WebXRImageTrackingModel, asPrimary: boolean = false) {
         if (!this.trackedImages.includes(image)) {
@@ -324,7 +341,11 @@ export class WebXRImageTracking extends Behaviour {
     }
 
 
-
+    /**
+     * List of images to track in the WebXR session. Use {@link WebXRImageTrackingModel} to define each image and the object to place on top of it.  
+     * 
+     * Use the `addImage()` and `setPrimaryImage()` methods to modify this array at runtime.
+     */
     @serializable(WebXRImageTrackingModel)
     readonly trackedImages: WebXRImageTrackingModel[] = [];
 
@@ -339,6 +360,7 @@ export class WebXRImageTracking extends Behaviour {
 
     private _supported: boolean = true;
 
+    /** @internal */
     awake(): void {
         if (debug) console.log(this)
         if (!this.trackedImages) return;
@@ -348,9 +370,11 @@ export class WebXRImageTracking extends Behaviour {
             }
         }
     }
+    /** @internal */
     onEnable() {
         USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport);
     }
+    /** @internal */
     onDisable(): void {
         USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport);
     }
@@ -362,7 +386,7 @@ export class WebXRImageTracking extends Behaviour {
     }
 
 
-
+    /** @internal */
     onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array<any> }): void {
         // console.log("onXRRequested", args, this.trackedImages)
         if (this.trackedImages) {
@@ -386,6 +410,7 @@ export class WebXRImageTracking extends Behaviour {
         }
     }
 
+    /** @internal */
     onEnterXR(_args: NeedleXREventArgs): void {
         if (this.trackedImages) {
             for (const trackedImage of this.trackedImages) {
@@ -408,6 +433,7 @@ export class WebXRImageTracking extends Behaviour {
         }
     };
 
+    /** @internal */
     onLeaveXR(_args: NeedleXREventArgs): void {
 
         if (!this.supported && DeviceUtilities.isAndroidDevice()) {
@@ -440,6 +466,7 @@ export class WebXRImageTracking extends Behaviour {
 
     private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the <a target=\"_blank\" href=\"#\" onclick=\"() => console.log('I')\">chrome://flags/#webxr-incubations</a> flag.";
 
+    /** @internal */
     onUpdateXR(args: NeedleXREventArgs): void {
         this.currentImages.length = 0;