@needle-tools/engine 3.0.1-alpha.5 → 3.1.0-alpha

package/src/engine-components/ParticleSystem.ts

@@ -193,6 +193,7 @@ class ParticleSystemEmissionOverTime extends BaseValueGenerator {
     }
 
     genValue(): number {
+        if (!this.system.isPlaying) return 0;
         if (!this.system.emission.enabled) return 0;
         if (this.system.currentParticles >= this.system.maxParticles) return 0;
         // emission over time
@@ -220,6 +221,7 @@ class ParticleSystemEmissionOverTime extends BaseValueGenerator {
 class ParticleSystemEmissionOverDistance extends BaseValueGenerator {
 
     genValue(): number {
+        if (!this.system.isPlaying) return 0;
         // this seems not be called yet
         return 0;
         // if (this.system.currentParticles >= this.system.maxParticles) return 0;
@@ -547,7 +549,7 @@ class ParticleSystemInterface implements ParticleSystemParameters {
         }
         switch (this.system.renderer.renderMode) {
             case ParticleSystemRenderMode.Billboard: return RenderMode.BillBoard;
-            // case ParticleSystemRenderMode.Stretch: return RenderMode.Stretch;
+            case ParticleSystemRenderMode.Stretch: return RenderMode.StretchedBillBoard;
             case ParticleSystemRenderMode.HorizontalBillboard: return RenderMode.LocalSpace;
             case ParticleSystemRenderMode.VerticalBillboard: return RenderMode.LocalSpace;
             case ParticleSystemRenderMode.Mesh: return RenderMode.LocalSpace;
@@ -597,8 +599,8 @@ export class ParticleSystem extends Behaviour implements IParticleSystem {
                 }
             }, true)
         }
+
         this._isPlaying = true;
-        this._time = 0;
 
         // https://github.com/Alchemist0823/three.quarks/pull/35
         if (this._particleSystem) {
@@ -608,12 +610,40 @@ export class ParticleSystem extends Behaviour implements IParticleSystem {
         this.emission?.reset();
     }
 
-    pause() {
+    pause(includeChildren = true) {
+        if (includeChildren) {
+            GameObject.foreachComponent(this.gameObject, comp => {
+                if (comp instanceof ParticleSystem && comp !== this) {
+                    comp.pause(false);
+                }
+            }, true)
+        }
         this._isPlaying = false;
     }
-    stop() {
+
+    /** clear=true removes all emitted particles */
+    stop(includeChildren = true, clear: boolean = false) {
+        if (includeChildren) {
+            GameObject.foreachComponent(this.gameObject, comp => {
+                if (comp instanceof ParticleSystem && comp !== this) {
+                    comp.stop(false, clear);
+                }
+            }, true)
+        }
         this._isPlaying = false;
         this._time = 0;
+        if (clear) this.reset();
+    }
+
+    /** remove emitted particles and reset time */
+    reset() {
+        this._time = 0;
+        if (this._particleSystem) {
+            this._particleSystem.particleNum = 0;
+            this._particleSystem["emissionState"].time = 0;
+            this._particleSystem["emitEnded"] = false;
+            this.emission?.reset();
+        }
     }
 
     private _state?: ParticlesEmissionState;

package/src/engine-components/ParticleSystemModules.ts

@@ -35,7 +35,7 @@ export interface IParticleSystem {
 
 export enum ParticleSystemRenderMode {
     Billboard = 0,
-    //   Stretch = 1,
+    Stretch = 1,
     HorizontalBillboard = 2,
     VerticalBillboard = 3,
     Mesh = 4,

package/src/engine-components/Skybox.ts

@@ -5,7 +5,7 @@ import { EXRLoader } from "three/examples/jsm/loaders/EXRLoader";
 import { EquirectangularRefractionMapping, sRGBEncoding, Texture, TextureLoader } from "three"
 import { syncField } from "../engine/engine_networking_auto";
 import { Camera } from "./Camera";
-import { getParam, getPath } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 
 const debug = getParam("debugskybox");
 
@@ -56,7 +56,7 @@ export class RemoteSkybox extends Behaviour {
         if(debug) console.log("Remote skybox url?: " + url);
 
         if (!url.startsWith("http") && !url.startsWith("www.") && !url.startsWith("data:")) {
-            url = getPath(this.sourceId, url);
+            url = resolveUrl(this.sourceId, url);
             if(debug) console.log("Remote skybox resolved to " + url);
         }
 

package/src/engine-components/VideoPlayer.ts

@@ -3,7 +3,7 @@ import * as THREE from "three";
 import { serializable } from "../engine/engine_serialization_decorator";
 import { LinearFilter, Material, Mesh, Object3D, RawShaderMaterial, ShaderMaterial, Texture, TextureLoader, Vector2, Vector4, VideoTexture } from "three";
 import { awaitInput } from "../engine/engine_input_utils";
-import { getParam } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 import { Renderer } from "./Renderer";
 import { getWorldScale } from "../engine/engine_three_utils";
 import { ObjectUtils, PrimitiveType } from "../engine/engine_create_objects";
@@ -49,11 +49,12 @@ export class VideoPlayer extends Behaviour {
     renderer: THREE.Object3D | null = null;
     @serializable()
     playOnAwake: boolean = true;
-    @serializable()
-    playOnEnable?: boolean;
 
     @serializable()
     aspectMode: AspectMode = AspectMode.None;
+    
+    @serializable(URL)
+    private clip?: string | MediaStream | null = null;
 
     @serializable()
     private renderMode?: VideoRenderMode;
@@ -153,7 +154,6 @@ export class VideoPlayer extends Behaviour {
     private audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.AudioSource;
 
     private source!: VideoSource;
-    private clip?: string | MediaStream | null = null;
     private url?: string | null = null;
 
     private _videoElement: HTMLVideoElement | null = null;
@@ -209,7 +209,7 @@ export class VideoPlayer extends Behaviour {
     }
 
     onEnable(): void {
-        if (this.playOnEnable === true) {
+        if (this.playOnAwake === true) {
             this.handleBeginPlaying(true);
         }
         if (this.screenspace) {
@@ -422,7 +422,7 @@ export class VideoPlayer extends Behaviour {
         let muted = !this._receivedInput && this.audioOutputMode !== VideoAudioOutputMode.None;
         if(!muted && this._muted) muted = true;
         this._videoElement.muted = muted;
-        if (this.playOnAwake || this.playOnEnable)
+        if (this.playOnAwake)
             this._videoElement.autoplay = true;
     }
 

package/src/engine-components/WebXR.ts

@@ -47,6 +47,7 @@ export enum WebXREvent {
     XRStopped = "xrStopped",
     XRUpdate = "xrUpdate",
     RequestVRSession = "requestVRSession",
+    ModifyAROptions = "modify-ar-options",
 }
 
 export declare type CreateButtonOptions = {
@@ -91,6 +92,9 @@ export class WebXR extends Behaviour {
         this.events.removeEventListener(type, listener);
         return listener;
     }
+    private static dispatchEvent(type: string, event: any): void {
+        this.events.dispatchEvent({ type, detail: event });
+    }
 
     public static createVRButton(webXR: WebXR, opts?: CreateButtonOptions): HTMLButtonElement | HTMLAnchorElement {
         if (!WebXR.XRSupported) {
@@ -122,7 +126,7 @@ export class WebXR extends Behaviour {
             console.warn("No dom overlay root found, HTML overlays on top of screen-based AR will not work.");
         }
 
-        const arButton = ARButton.createButton(webXR.context.renderer, options);
+        const arButton = ARButton.createButton(webXR.context.renderer, options, this.onModifyAROptions.bind(this));
         arButton.classList.add('webxr-ar-button');
         arButton.classList.add('webxr-button');
         WebXR.resetButtonStyles(arButton);
@@ -131,6 +135,10 @@ export class WebXR extends Behaviour {
         return arButton;
     }
 
+    private static onModifyAROptions(options){
+        WebXR.dispatchEvent(WebXREvent.ModifyAROptions, options);
+    }
+
     public static resetButtonStyles(button) {
         if (!button) return;
         button.style.position = "";
@@ -233,7 +241,8 @@ export class WebXR extends Behaviour {
         this.context.domElement.append(buttonsContainer);
 
         // AR support
-        if (this.enableAR && this.createARButton && arSupported) {
+        // if (this.enableAR && this.createARButton && arSupported) 
+        {
             arButton = WebXR.createARButton(this);
             this._arButton = arButton;
             buttonsContainer.appendChild(arButton);

package/src/engine-components/WebXRController.ts

@@ -8,7 +8,7 @@ import { InstancingUtil } from "../engine/engine_instancing";
 import { Mathf } from "../engine/engine_math";
 import { RaycastOptions } from "../engine/engine_physics";
 import { getWorldPosition, getWorldQuaternion, setWorldPosition, setWorldQuaternion } from "../engine/engine_three_utils";
-import { getParam, getPath } from "../engine/engine_utils";
+import { getParam, resolveUrl } from "../engine/engine_utils";
 import { addDracoAndKTX2Loaders } from "../engine/engine_loaders";
 
 import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt";
@@ -85,7 +85,7 @@ export class WebXRController extends Behaviour {
         const loader = new GLTFLoader();
         addDracoAndKTX2Loaders(loader, context);
         if (ctrl.webXR.handModelPath && ctrl.webXR.handModelPath !== "")
-            loader.setPath(getPath(owner.sourceId, ctrl.webXR.handModelPath));
+            loader.setPath(resolveUrl(owner.sourceId, ctrl.webXR.handModelPath));
         else
             // from XRHandMeshModel.js
             loader.setPath('https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles/generic-hand/');

package/src/engine-components/WebXRImageTracking.ts

@@ -0,0 +1,192 @@
+import { WebXR } from "./WebXR";
+import { serializable } from "../engine/engine_serialization";
+import { Behaviour } from "./Component";
+import { Matrix4, Object3D, Quaternion, Vector, Vector3 } from "three";
+import { CircularBuffer, getParam } from "../engine/engine_utils";
+
+// https://github.com/immersive-web/marker-tracking/blob/main/explainer.md
+
+const debug = getParam("debugimagetracking");
+
+const _scaleTemp = new Vector3();
+
+export class WebXRTrackedImage {
+
+
+    get url(): string { return this._trackedImage.image ?? ""; }
+    get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; }
+    get bitmap(): ImageBitmap { return this._bitmap; }
+    readonly measuredSize: number;
+    readonly state: "tracked" | "emulated";
+
+    // private _matrix: Matrix4 | null = null;
+    // private get matrix(): Matrix4 {
+    //     if (!this._matrix) {
+    //         // this._matrix = WebXRTrackedImage._matrixBuffer.get();
+    //         // const matrix = this._pose.transform.matrix;
+    //         // this._matrix.fromArray(matrix);
+    //     }
+    //     return this._matrix!;
+    // }
+
+    /** Copy the image position to a vector */
+    getPosition(vec: Vector3) {
+        this.ensureTransformData();
+        vec.copy(this._position);
+        return vec;
+    }
+
+    /** Copy the image rotation to a quaternion */
+    getQuaternion(quat: Quaternion) {
+        this.ensureTransformData();
+        quat.copy(this._rotation);
+        return quat;
+    }
+
+    applyToObject(object: Object3D) {
+        this.ensureTransformData();
+        object.position.copy(this._position);
+        object.quaternion.copy(this._rotation);
+    }
+
+    // private static _matrixBuffer: CircularBuffer<Matrix4> = new CircularBuffer(() => new Matrix4(), 20);
+    private static _positionBuffer: CircularBuffer<Vector3> = new CircularBuffer(() => new Vector3(), 20);
+    private static _rotationBuffer: CircularBuffer<Quaternion> = new CircularBuffer(() => new Quaternion(), 20);
+    private _position!: Vector3;
+    private _rotation!: Quaternion;
+    private ensureTransformData() {
+        if (!this._position) {
+            this._position = WebXRTrackedImage._positionBuffer.get();
+            this._rotation = WebXRTrackedImage._rotationBuffer.get();
+            const t = this._pose.transform;
+            this._position.set(-t.position.x, t.position.y, -t.position.z);
+            this._rotation.set(-t.orientation.x, t.orientation.y, -t.orientation.z, t.orientation.w);
+        }
+    }
+
+    private readonly _trackingComponent: WebXRImageTracking;;
+    private readonly _trackedImage: WebXRImageTrackingModel;
+    private readonly _bitmap: ImageBitmap;
+    private readonly _pose: any;
+
+    constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) {
+        this._trackingComponent = context;;
+        this._trackedImage = trackedImage;
+        this._bitmap = bitmap;
+        this.measuredSize = measuredSize;
+        this.state = state;
+        this._pose = pose;
+    }
+
+}
+
+declare type WebXRImageTrackingEvent = (images: WebXRImageTrackingEvent[]) => void;
+
+export class WebXRImageTrackingModel {
+
+    @serializable()
+    image?: string;
+
+    @serializable()
+    widthInMeters!: number;
+
+}
+
+export class WebXRImageTracking extends Behaviour {
+
+    @serializable(WebXRImageTrackingModel)
+    trackedImages!: WebXRImageTrackingModel[];
+
+
+
+
+    private readonly trackedImageIndexMap: Map<number, WebXRImageTrackingModel> = new Map();
+
+    private static _imageElements: Map<string, ImageBitmap | null> = new Map();
+
+    awake(): void {
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image) {
+                if (WebXRImageTracking._imageElements.has(trackedImage.image)) {
+                }
+                else {
+                    const url = trackedImage.image;
+                    WebXRImageTracking._imageElements.set(url, null);
+                    const imageElement = document.createElement("img") as HTMLImageElement;
+                    imageElement.src = url;
+                    imageElement.addEventListener("load", async () => {
+                        const img = await createImageBitmap(imageElement);
+                        WebXRImageTracking._imageElements.set(url, img);
+                    });
+                }
+            }
+        }
+    }
+
+    onEnable(): void {
+        WebXR.addEventListener("modify-ar-options", this.onModifyAROptions);
+    }
+
+    onDisable(): void {
+        WebXR.removeEventListener("modify-ar-options", this.onModifyAROptions);
+    }
+
+
+    private onModifyAROptions = (event: any) => {
+        const options = event.detail;
+        const features = options.optionalFeatures || [];
+        features.push("image-tracking");
+        options.optionalFeatures = features;
+
+        options.trackedImages = [];
+        for (const trackedImage of this.trackedImages) {
+            if (trackedImage.image?.length && trackedImage.widthInMeters > 0) {
+                const bitmap = WebXRImageTracking._imageElements.get(trackedImage.image);
+                if (bitmap) {
+                    this.trackedImageIndexMap.set(options.trackedImages.length, trackedImage);
+                    options.trackedImages.push({
+                        image: bitmap,
+                        widthInMeters: trackedImage.widthInMeters
+                    });
+                }
+            }
+        }
+    }
+
+    onBeforeRender(frame: XRFrame | null): void {
+        //@ts-ignore
+        if (frame?.session && typeof frame.getImageTrackingResults === "function") {
+            //@ts-ignore
+            const results = frame.getImageTrackingResults();
+            if (results.length) {
+                const space = this.context.renderer.xr.getReferenceSpace();
+                if (space) {
+                    const images: WebXRTrackedImage[] = [];
+                    for (const result of results) {
+                        const imageIndex = result.index;
+                        const trackedImage = this.trackedImageIndexMap.get(imageIndex);
+                        if (trackedImage) {
+                            const pose = frame.getPose(result.imageSpace, space);
+                            const state = result.trackingState;
+                            const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose);
+                            images.push(imageData);
+                        }
+                        else {
+                            if (debug) {
+                                console.warn("No tracked image for index", imageIndex);
+                            }
+                        }
+                    }
+                    if (images.length > 0) {
+                        try {
+                            this.dispatchEvent(new CustomEvent("image-tracking", { detail: images }));
+                        }
+                        catch (e) {
+                            console.error(e);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

package/src/engine-components/codegen/components.ts

@@ -145,6 +145,7 @@ export { TestRunner } from "../TestRunner";
 export { TestSimulateUserData } from "../TestRunner";
 export { Text } from "../ui/Text";
 export { TextureSheetAnimationModule } from "../ParticleSystemModules";
+export { TiltShiftEffect } from "../postprocessing/Effects/TiltShiftEffect";
 export { ToneMapping } from "../postprocessing/Effects/Tonemapping";
 export { TrailModule } from "../ParticleSystemModules";
 export { TransformData } from "../export/usdz/extensions/Animation";
@@ -168,7 +169,10 @@ export { WebARSessionRoot } from "../WebARSessionRoot";
 export { WebXR } from "../WebXR";
 export { WebXRAvatar } from "../WebXRAvatar";
 export { WebXRController } from "../WebXRController";
+export { WebXRImageTracking } from "../WebXRImageTracking";
+export { WebXRImageTrackingModel } from "../WebXRImageTracking";
 export { WebXRSync } from "../WebXRSync";
+export { WebXRTrackedImage } from "../WebXRImageTracking";
 export { XRFlag } from "../XRFlag";
 export { XRGrabModel } from "../WebXRGrabRendering";
 export { XRGrabRendering } from "../WebXRGrabRendering";

package/src/engine-components/postprocessing/Effects/DepthOfField.ts

@@ -36,6 +36,9 @@ export class DepthOfField extends PostProcessingEffect {
     @serializable(VolumeParameter)
     resolutionScale?: VolumeParameter;
 
+    @serializable(VolumeParameter)
+    bokehScale?: VolumeParameter;
+
     init() {
         this.focalLength.valueProcessor = v => {
             const t = v / 300;

package/src/engine-components/postprocessing/Effects/TiltShiftEffect.ts

@@ -0,0 +1,56 @@
+import { registerCustomEffectType } from "../VolumeProfile";
+import { EffectProviderResult, PostProcessingEffect } from "../PostProcessingEffect";
+import { KernelSize, TiltShiftEffect as TiltShift } from "postprocessing";
+import { VolumeParameter } from "../VolumeParameter";
+import { serializable } from "../../../engine/engine_serialization";
+
+
+export class TiltShiftEffect extends PostProcessingEffect {
+    get typeName(): string {
+        return "TiltShiftEffect";
+    }
+
+    @serializable(VolumeParameter)
+    offset!: VolumeParameter;
+    @serializable(VolumeParameter)
+    rotation!: VolumeParameter;
+    @serializable(VolumeParameter)
+    focusArea!: VolumeParameter;
+    @serializable(VolumeParameter)
+    feather!: VolumeParameter
+    @serializable(VolumeParameter)
+    kernelSize!: VolumeParameter;
+    @serializable(VolumeParameter)
+    resolutionScale!: VolumeParameter;
+
+    init(): void {
+        this.offset.defaultValue = 0;
+        this.rotation.defaultValue = 0;
+        this.focusArea.defaultValue = 0.4;
+        this.feather.defaultValue = 0.3;
+        this.kernelSize.defaultValue = KernelSize.MEDIUM;
+        this.resolutionScale.defaultValue = 1 / window.devicePixelRatio;
+    }
+
+
+    onCreateEffect(): EffectProviderResult | undefined {
+
+        console.log(this);
+
+        const effect = new TiltShift({
+            kernelSize: KernelSize.VERY_LARGE,
+        });
+        
+        this.offset.onValueChanged = v => effect.offset = v;
+        this.rotation.onValueChanged = v => effect.rotation = v;
+        this.focusArea.onValueChanged = v => effect.focusArea = v;
+        this.feather.onValueChanged = v => effect.feather = v;
+        this.kernelSize.onValueChanged = v => effect.blurPass.kernelSize = v;
+        this.resolutionScale.onValueChanged = v => effect.resolution.scale = v / window.devicePixelRatio;
+
+
+        return effect;
+    }
+
+}
+registerCustomEffectType("TiltShiftEffect", TiltShiftEffect);

package/src/engine-components/timeline/TimelineTracks.ts

@@ -5,7 +5,7 @@ import { GameObject } from "../Component";
 import { Context } from "../../engine/engine_setup";
 import { SignalReceiver } from "./SignalAsset";
 import { AnimationClip, Quaternion, Vector3 } from "three";
-import { getParam, getPath } from "../../engine/engine_utils";
+import { getParam, resolveUrl } from "../../engine/engine_utils";
 import { AudioSource } from "../AudioSource";
 import { Animator } from "../Animator"
 
@@ -517,7 +517,7 @@ export class AudioTrackHandler extends TrackHandler {
     private getAudioFilePath(path: string) {
         // TODO: this should be the timeline asset location probably which MIGHT be different
         const glbLocation = this.director.sourceId;
-        return getPath(glbLocation, path);
+        return resolveUrl(glbLocation, path);
     }
 
     onAllowAudioChanged(allow: boolean) {

package/src/engine-components/ui/Text.ts

@@ -7,7 +7,7 @@ import { FrameEvent } from '../../engine/engine_setup';
 import { updateRenderSettings } from './Utils';
 import { Canvas } from './Canvas';
 import { serializable } from '../../engine/engine_serialization_decorator';
-import { getParam, getPath } from '../../engine/engine_utils';
+import { getParam, resolveUrl } from '../../engine/engine_utils';
 
 const debug = getParam("debugtext");
 
@@ -481,24 +481,24 @@ export class Text extends Graphic {
 
         // if a font path has a known suffix we remove it
         if (fontName.endsWith("-regular")) {
-            if (style === FontStyle.Normal) return getPath(this.sourceId, fontName);
+            if (style === FontStyle.Normal) return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-regular".length);
         }
         else if (fontName.endsWith("-bold")) {
-            if (style === FontStyle.Bold)return getPath(this.sourceId, fontName);
+            if (style === FontStyle.Bold)return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-bold".length);
         }
         else if (fontName.endsWith("-italic")) {
-            if (style === FontStyle.Italic)return getPath(this.sourceId, fontName);
+            if (style === FontStyle.Italic)return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-italic".length);
         }
         else if (fontName.endsWith("-bolditalic")) {
-            if (style === FontStyle.BoldAndItalic)return getPath(this.sourceId, fontName);
+            if (style === FontStyle.BoldAndItalic)return resolveUrl(this.sourceId, fontName);
                 fontName = fontName.substring(0, fontName.length - "-bolditalic".length);
         }
         else
             // If a font does not have a specific style suffic we dont support getting the correct font style
-            return getPath(this.sourceId, fontName);
+            return resolveUrl(this.sourceId, fontName);
 
         switch (style) {
             case FontStyle.Normal:
@@ -515,7 +515,7 @@ export class Text extends Graphic {
                 break;
         }
 
-        return getPath(this.sourceId, fontName);
+        return resolveUrl(this.sourceId, fontName);
     }
 }
 

package/src/include/three/ARButton.js

@@ -1,6 +1,6 @@
 class ARButton {
 
-	static createButton( renderer, options = {} ) {
+	static createButton( renderer, options = {}, beforeRequestSession ) {
 
 		const button = document.createElement( 'button' );
 		let ARButtonControlsDomOverlay = false;
@@ -126,7 +126,7 @@ class ARButton {
 			button.onclick = function () {
 
 				if ( currentSession === null ) {
-
+					beforeRequestSession?.call(this, options);
 					navigator.xr.requestSession( 'immersive-ar', options ).then( onSessionStarted );
 
 				} else {