@needle-tools/engine 3.27.4-beta → 3.28.0-beta

package/src/engine-components/export/usdz/ThreeUSDZExporter.ts

@@ -1,5 +1,7 @@
 import { Renderer } from '../../Renderer.js';
 import { GameObject } from '../../Component.js';
+import type { OffscreenCanvasExt } from '../../../engine/engine_shims.js';
+import '../../../engine/engine_shims.js';
 import {
 	PlaneGeometry,
 	Texture,
@@ -24,7 +26,8 @@ import {
 	SRGBColorSpace,
 	AnimationClip,
 	OrthographicCamera,
-	BufferAttribute
+	BufferAttribute,
+	Vector4
 } from 'three';
 import * as fflate from 'three/examples/jsm/libs/fflate.module.js';
 
@@ -77,12 +80,6 @@ function findStructuralNodesInBoneHierarchy(bones: Array<Object3D>) {
 	return structuralNodes;
 }
 
-// TODO: remove once we update to TypeScript 5 that has proper types for OffscreenCanvas
-declare type OffsetCanvasExt = OffscreenCanvas & {
-	convertToBlob: (options?: any) => Promise<Blob>;
-
-}
-
 class USDObject {
 
 	static USDObject_export_id = 0;
@@ -432,13 +429,15 @@ class USDWriter {
 
 }
 
+declare type TextureMap = {[name: string]: {texture: Texture, scale?: Vector4}};
+
 class USDZExporterContext {
 	root: any;
 	exporter: any;
 	extensions: any;
-	materials: {};
-	textures: {};
-	files: {};
+	materials: Map<string, Material>;
+	textures: TextureMap;
+	files: { [path: string]: Uint8Array | [Uint8Array, fflate.ZipOptions] | null | any }
 	document: USDDocument;
 	output: string;
 	animations: AnimationClip[];
@@ -451,7 +450,7 @@ class USDZExporterContext {
 		if ( extensions )
 			this.extensions = extensions;
 
-		this.materials = {};
+		this.materials = new Map();
 		this.textures = {};
 		this.files = {};
 		this.document = new USDDocument();
@@ -582,18 +581,19 @@ class USDZExporter {
 
 		for ( const id in textures ) {
 
-			let texture = textures[ id ];
+			const textureData = textures[ id ];
+			let texture = textureData.texture;
 
 			const isRGBA = formatsWithAlphaChannel.includes( texture.format );
 
+			//@ts-ignore
 			if ( texture.isCompressedTexture || texture.isRenderTargetTexture ) {
 
 				texture = decompressGpuTexture( texture, options.maxTextureSize, decompressionRenderer );
 
 			}
 			
-			// TODO add readback options for textures that don't have texture.image
-			const canvas = await imageToCanvas( texture.image ).catch( err => {
+			const canvas = await imageToCanvas( texture.image, textureData.scale, false, options.maxTextureSize ).catch( err => {
 				console.error("Error converting texture to canvas", texture, err);
 			});
 
@@ -873,6 +873,7 @@ function decompressGpuTexture( texture, maxTextureSize = Infinity, renderer: Web
                 #else
                 gl_FragColor = texture2D( blitTexture, vUv);
                 #endif
+				gl_FragColor.rgb *= gl_FragColor.a;
             }`
 	} );
 
@@ -930,7 +931,7 @@ function isImageBitmap( image ) {
 
 }
 
-async function imageToCanvas( image, color: string | undefined = undefined, flipY = false, maxTextureSize = 4096 ) {
+async function imageToCanvas( image, color: Vector4 | undefined = undefined, flipY = false, maxTextureSize = 4096 ) {
 
 	if ( isImageBitmap( image ) ) {
 
@@ -951,14 +952,13 @@ async function imageToCanvas( image, color: string | undefined = undefined, flip
 
 		context.drawImage( image, 0, 0, canvas.width, canvas.height );
 
-		// TODO remove, not used anymore
+		// Currently only used to apply opacity scale since QuickLook and usdview don't support that yet
 		if ( color !== undefined ) {
 
-			const hex = parseInt( color, 16 );
-
-			const r = ( hex >> 16 & 255 ) / 255;
-			const g = ( hex >> 8 & 255 ) / 255;
-			const b = ( hex & 255 ) / 255;
+			const r = color.x;
+			const g = color.y;
+			const b = color.z;
+			const a = color.w;
 
 			const imagedata = context.getImageData( 0, 0, canvas.width, canvas.height );
 			const data = imagedata.data;
@@ -968,6 +968,7 @@ async function imageToCanvas( image, color: string | undefined = undefined, flip
 				data[ i + 0 ] = data[ i + 0 ] * r;
 				data[ i + 1 ] = data[ i + 1 ] * g;
 				data[ i + 2 ] = data[ i + 2 ] * b;
+				data[ i + 3 ] = data[ i + 3 ] * a;
 
 			}
 
@@ -975,7 +976,7 @@ async function imageToCanvas( image, color: string | undefined = undefined, flip
 
 		}
 
-		return canvas as OffsetCanvasExt;
+		return canvas as OffscreenCanvasExt;
 
 	} else {
 
@@ -1375,7 +1376,7 @@ function buildVector2Array( attribute, count ) {
 
 // Materials
 
-function buildMaterials( materials, textures, quickLookCompatible = false ) {
+function buildMaterials( materials: Map<string, Material>, textures: TextureMap, quickLookCompatible = false ) {
 
 	const array: Array<string> = [];
 
@@ -1396,7 +1397,7 @@ ${array.join( '' )}
 
 }
 
-function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatible = false ) {
+function buildMaterial( material: MeshBasicMaterial, textures: TextureMap, quickLookCompatible = false ) {
 
 	// https://graphics.pixar.com/usd/docs/UsdPreviewSurface-Proposal.html
 
@@ -1408,7 +1409,11 @@ function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatib
 
 		const id = texture.id + ( color ? '_' + color.getHexString() : '' ) + ( opacity !== undefined ? '_' + opacity : '' );
 
-		textures[ id ] = texture;
+		// Seems neither QuickLook nor usdview support scale/bias on .a values, so we need to bake opacity multipliers into
+		// the texture. This is not ideal.
+		const opacityIsAppliedToTextureAndNotAsScale = quickLookCompatible && opacity !== undefined;
+		const scaleToApply = opacityIsAppliedToTextureAndNotAsScale ? new Vector4(1, 1, 1, opacity) : undefined;
+		textures[ id ] = { texture, scale: scaleToApply };
 		
 		const uv = texture.channel > 0 ? 'st' + texture.channel : 'st';
 
@@ -1485,7 +1490,7 @@ function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatib
 			token inputs:sourceColorSpace = "${ texture.colorSpace === 'srgb' ? 'sRGB' : 'raw' }"
             float2 inputs:st.connect = ${needsTextureTransform ? textureTransformOutput : textureTransformInput}
 			${needsTextureScale ? `
-			float4 inputs:scale = (${color ? color.r + ', ' + color.g + ', ' + color.b : '1, 1, 1'}, ${opacity !== undefined ? opacity : '1'})
+			float4 inputs:scale = (${color ? color.r + ', ' + color.g + ', ' + color.b : '1, 1, 1'}, ${(opacity !== undefined && !opacityIsAppliedToTextureAndNotAsScale) ? opacity : '1'})
 			` : `` }
 			${needsNormalScaleAndBias ? `
 			float4 inputs:scale = (${normalScaleValueString}, ${normalScaleValueString}, ${normalScaleValueString}, 1)
@@ -1503,7 +1508,7 @@ function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatib
 	}
 
 	let effectiveOpacity = ( material.transparent || material.alphaTest ) ? material.opacity : 1;
-
+	
 	if ( material instanceof MeshPhysicalMaterial && material.transmission !== undefined) {
 
 		// TODO does not help when a roughnessMap is used
@@ -1553,7 +1558,7 @@ function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatib
 		inputs.push( `${pad}float inputs:opacity.connect = ${materialRoot}/Material_${material.id}/Texture_${material.alphaMap.id}_opacity.outputs:r>` );
 		inputs.push( `${pad}float inputs:opacityThreshold = 0.0001` );
 
-		samplers.push( buildTexture( material.alphaMap, 'opacity' ) );
+		samplers.push( buildTexture( material.alphaMap, 'opacity', new Color( 1, 1, 1 ), effectiveOpacity ) );
 
 	} else {
 
@@ -1572,12 +1577,15 @@ function buildMaterial( material: MeshBasicMaterial, textures, quickLookCompatib
 		if ( material.emissiveMap ) {
 
 			inputs.push( `${pad}color3f inputs:emissiveColor.connect = ${materialRoot}/Material_${material.id}/Texture_${material.emissiveMap.id}_emissive.outputs:rgb>` );
-
-			samplers.push( buildTexture( material.emissiveMap, 'emissive' ) );
+			const color = material.emissive.clone();
+			color.multiplyScalar( material.emissiveIntensity );
+			samplers.push( buildTexture( material.emissiveMap, 'emissive', color ) );
 
 		} else if ( material.emissive?.getHex() > 0 ) {
 
-			inputs.push( `${pad}color3f inputs:emissiveColor = ${buildColor( material.emissive )}` );
+			const color = material.emissive.clone();
+			color.multiplyScalar( material.emissiveIntensity );
+			inputs.push( `${pad}color3f inputs:emissiveColor = ${buildColor( color )}` );
 
 		} else {
 			

package/src/engine-components/export/usdz/extensions/behavior/BehaviourComponents.ts

@@ -477,11 +477,7 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
         ensureRaycaster(this.gameObject);
     }
 
-    onPointerClick(args: PointerEventData) {
-        args.use();
-
-        if (!this.clip) return;
-
+    ensureAudioSource() {
         if (!this.target) {
             const newAudioSource = this.gameObject.addNewComponent(AudioSource);
             if (newAudioSource) {
@@ -491,6 +487,14 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
                 newAudioSource.loop = false;
             }
         }
+    }
+
+    onPointerClick(args: PointerEventData) {
+        args.use();
+
+        if (!this.clip) return;
+
+        this.ensureAudioSource();
 
         if (this.target) {
 
@@ -519,11 +523,21 @@ export class PlayAudioOnClick extends Behaviour implements IPointerClickHandler,
             const clipName = clipUrl.split("/").pop();
             const volume = this.target ? this.target.volume : 1;
             const auralMode = this.target && this.target.spatialBlend == 0 ? AuralMode.NonSpatial : AuralMode.Spatial;
-            const playClip = new BehaviorModel("playAudio " + this.name,
+            
+            const playClipOnTap = new BehaviorModel("playAudio " + this.name,
                 TriggerBuilder.tapTrigger(this.gameObject),
                 ActionBuilder.playAudioAction(playbackTarget, "audio/" + clipName, PlayAction.Play, volume, auralMode),
             );
-            ext.addBehavior(playClip);
+            ext.addBehavior(playClipOnTap);
+
+            // automatically play audio on start too if the referenced AudioSource has playOnAwake enabled
+            if (this.target && this.target.playOnAwake && this.target.enabled) {
+                const playClipOnStart = new BehaviorModel("playAudioOnStart " + this.name,
+                    TriggerBuilder.sceneStartTrigger(),
+                    ActionBuilder.playAudioAction(playbackTarget, "audio/" + clipName, PlayAction.Play, volume, auralMode),
+                );
+                ext.addBehavior(playClipOnStart);
+            }
         }
     }
 

package/src/engine-components/ui/EventSystem.ts

@@ -7,7 +7,7 @@ import { Context } from "../../engine/engine_setup.js";
 import { type IPointerEventHandler, PointerEventData, hasPointerEventComponent } from "./PointerEvents.js";
 import { ObjectRaycaster, Raycaster } from "./Raycaster.js";
 import { InputEvents, NEPointerEvent, PointerType } from "../../engine/engine_input.js";
-import { Object3D } from "three";
+import { Mesh, Object3D } from "three";
 import type { ICanvasGroup } from "./Interfaces.js";
 import { getParam } from "../../engine/engine_utils.js";
 import { UIRaycastUtils } from "./RaycastUtils.js";
@@ -295,6 +295,14 @@ export class EventSystem extends Behaviour {
      * */
     private shouldRaycastObject = (obj: Object3D): RaycastTestObjectReturnType => {
 
+        if (!(obj instanceof Mesh)) 
+            return "continue in children";
+
+        // Layer 2 is IgnoreRaycast
+        if (obj.layers.isEnabled(2)) {
+            return "continue in children"
+        }
+
         // check if this object is actually a UI shadow hierarchy object
         let shadowComponent: Object3D | null = null;
         const isUI = isUIObject(obj);

package/src/engine/dist/api.js

@@ -1,73 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !exports.hasOwnProperty(p)) __createBinding(exports, m, p);
-};
-exports.__esModule = true;
-__exportStar(require("./extensions/index.js"), exports);
-__exportStar(require("./engine_addressables.js"), exports);
-__exportStar(require("./engine_application.js"), exports);
-__exportStar(require("./engine_assetdatabase.js"), exports);
-__exportStar(require("./engine_create_objects.js"), exports);
-__exportStar(require("./engine_components_internal.js"), exports);
-__exportStar(require("./engine_components.js"), exports);
-__exportStar(require("./engine_components_internal.js"), exports);
-__exportStar(require("./engine_context_registry.js"), exports);
-__exportStar(require("./engine_context.js"), exports);
-__exportStar(require("./engine_coroutine.js"), exports);
-__exportStar(require("./engine_constants.js"), exports);
-__exportStar(require("./debug/index.js"), exports);
-__exportStar(require("./engine_element.js"), exports);
-__exportStar(require("./engine_element_loading.js"), exports);
-__exportStar(require("./engine_element_attributes.js"), exports);
-var engine_gizmos_js_1 = require("./engine_gizmos.js");
-__createBinding(exports, engine_gizmos_js_1, "Gizmos");
-__exportStar(require("./engine_gltf.js"), exports);
-__exportStar(require("./engine_hot_reload.js"), exports);
-__exportStar(require("./engine_gameobject.js"), exports);
-__exportStar(require("./engine_networking.js"), exports);
-__exportStar(require("./engine_networking_types.js"), exports);
-var engine_networking_auto_js_1 = require("./engine_networking_auto.js");
-__createBinding(exports, engine_networking_auto_js_1, "syncField");
-__exportStar(require("./engine_networking_files.js"), exports);
-__exportStar(require("./engine_networking_instantiate.js"), exports);
-__exportStar(require("./engine_networking_streams.js"), exports);
-__exportStar(require("./engine_networking_utils.js"), exports);
-__exportStar(require("./engine_networking_peer.js"), exports);
-__exportStar(require("./engine_patcher.js"), exports);
-__exportStar(require("./engine_playerview.js"), exports);
-__exportStar(require("./engine_physics.js"), exports);
-__exportStar(require("./engine_physics.types.js"), exports);
-__exportStar(require("./engine_physics_rapier.js"), exports);
-__exportStar(require("./engine_scenelighting.js"), exports);
-__exportStar(require("./engine_input.js"), exports);
-__exportStar(require("./engine_math.js"), exports);
-__exportStar(require("./js-extensions/index.js"), exports);
-__exportStar(require("./engine_scenetools.js"), exports);
-__exportStar(require("./engine_serialization.js"), exports);
-var engine_serialization_core_js_1 = require("./engine_serialization_core.js");
-__createBinding(exports, engine_serialization_core_js_1, "type");
-__exportStar(require("./engine_texture.js"), exports);
-__exportStar(require("./engine_three_utils.js"), exports);
-__exportStar(require("./engine_time.js"), exports);
-__exportStar(require("./engine_types.js"), exports);
-__exportStar(require("./engine_utils_screenshot.js"), exports);
-__exportStar(require("./engine_web_api.js"), exports);
-__exportStar(require("./engine_utils.js"), exports);
-var engine_typestore_js_1 = require("./engine_typestore.js");
-__createBinding(exports, engine_typestore_js_1, "TypeStore");
-__createBinding(exports, engine_typestore_js_1, "registerType");
-var engine_instancing_js_1 = require("./engine_instancing.js");
-__createBinding(exports, engine_instancing_js_1, "InstancingUtil");
-var engine_util_decorator_js_1 = require("./engine_util_decorator.js");
-__createBinding(exports, engine_util_decorator_js_1, "validate");
-__createBinding(exports, engine_util_decorator_js_1, "prefix");
-var engine_license_js_1 = require("./engine_license.js");
-__createBinding(exports, engine_license_js_1, "hasProLicense");
-__createBinding(exports, engine_license_js_1, "hasIndieLicense");

package/src/engine/dist/api.js.meta

@@ -1,7 +0,0 @@
-fileFormatVersion: 2
-guid: f0e01709ea200b648a0c053c2c79f225
-DefaultImporter:
-  externalObjects: {}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: