@needle-tools/engine 2.65.2-pre → 2.67.0-pre

package/src/engine-components/postprocessing/PostProcessingEffect.ts

@@ -0,0 +1,112 @@
+import { serializable } from "../../engine/engine_serialization";
+import { Effect, Pass } from "postprocessing";
+import { VolumeParameter } from "./VolumeParameter";
+import { Component } from "../Component";
+import { ISerializable, SerializationContext } from "../../engine/engine_serialization_core";
+import { EditorModification, IEditorModification } from "../../engine/engine_editor-sync";
+
+export declare type EffectProviderResult = Effect | Pass | Array<Effect | Pass>;
+
+export interface IEffectProvider {
+    apply(): void | undefined | EffectProviderResult;
+    unapply(): void;
+}
+
+export abstract class PostProcessingEffect extends Component implements IEffectProvider, ISerializable, IEditorModification {
+
+    abstract get typeName(): string;
+
+    onEnable(): void {
+        // Dont override the serialized value by enabling (we could also just disable this component / map enabled to active)
+        if (this.__internalDidAwakeAndStart)
+            this.active = true;
+    }
+
+    onDisable(): void {
+        this.active = false;
+    }
+
+    @serializable()
+    active: boolean = true;
+
+    /** override to initialize bindings on parameters */
+    init() {
+
+    }
+
+    /** previously created effect (if any) */
+    private _result: void | undefined | EffectProviderResult;
+
+
+    /** Apply post settings. Make sure to call super.apply() if you also create an effect */
+    apply(): void | undefined | EffectProviderResult {
+        if (!this._result) {
+            this._result = this.onCreateEffect?.call(this);
+            if (this._result) {
+                this.initParameters();
+            }
+        }
+        return this._result;
+    }
+
+    /** Reset previously set values (e.g. when adjusting settings on the renderer like Tonemapping) */
+    unapply(): void { }
+
+    /** implement to create a effect once to be cached in the base class. Make sure super.apply() is called if you also override apply */
+    onCreateEffect?(): EffectProviderResult | undefined
+
+    dispose() {
+        if (this._result) {
+            if (Array.isArray(this._result)) {
+                this._result.forEach(r => r.dispose());
+            } else {
+                this._result.dispose();
+            }
+        }
+        this._result = undefined;
+    }
+
+    private initParameters() {
+        // Automatically call init on all VolumeParameter properties
+        // This will enforce the valueProcessor and onValueChanged to be called
+        const keys = Object.keys(this);
+        for (const key of keys) {
+            const value = this[key];
+            if (value instanceof VolumeParameter) {
+                value.__init();
+            }
+        }
+    }
+
+    onAfterDeserialize(data: any, _context: SerializationContext): void {
+        // When using additional effects and parameters exported from the editor are not in the volume parameter format
+        if (typeof data === "object") {
+            const types = this["$serializedTypes"];
+            if (types) {
+                for (const fieldName in Object.keys(types)) {
+                    const type = types[fieldName];
+                    if (type === VolumeParameter) {
+                        const value = data[fieldName];
+                        if (value !== undefined) {
+                            this[fieldName].value = value;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    onEditorModification(modification: EditorModification): void | boolean | undefined {
+        // Handle a property modification if the property is a VolumeParameter and the modification is just a plain value
+        const key = modification.propertyName;
+        if (this[key] instanceof VolumeParameter) {
+            const value = modification.value;
+            this[key].value = value;
+            return true;
+        }
+    }
+}
+
+
+

package/src/engine-components/postprocessing/PostProcessingHandler.ts

@@ -0,0 +1,148 @@
+import { HalfFloatType, sRGBEncoding, WebGLRenderTarget } from "three";
+import { Context } from "../../engine/engine_setup";
+import { getParam } from "../../engine/engine_utils";
+import { Effect, EffectComposer, EffectPass, Pass, RenderPass } from "postprocessing";
+import { showBalloonWarning } from "../../engine/debug/debug";
+import { Camera } from "../Camera";
+import { PostProcessingEffect } from "./PostProcessingEffect";
+
+const debug = getParam("debugpost");
+
+const activeKey = Symbol("postprocessing-handler");
+
+export class PostProcessingHandler {
+
+    private _composer: EffectComposer | null = null;
+    private _lastVolumeComponents?: PostProcessingEffect[];
+    private _effects: Array<Effect | Pass> = [];
+
+
+    apply(context: Context, components: PostProcessingEffect[]) {
+        this.onApply(context, components);
+    }
+
+    unapply(context: Context) {
+        if (this._lastVolumeComponents) {
+            for (const component of this._lastVolumeComponents) {
+                component.unapply();
+            }
+            this._lastVolumeComponents.length = 0;
+        }
+        const active = context[activeKey] as PostProcessingHandler | null;
+        if (active === this) {
+            delete context[activeKey];
+        }
+        if (context.composer === this._composer)
+            context.composer = null;
+    }
+
+    dispose(context: Context) {
+        this.unapply(context);
+
+        for (const effect of this._effects) {
+            effect.dispose();
+        }
+        this._effects.length = 0;
+    }
+
+
+    private onApply(context: Context, components: PostProcessingEffect[]) {
+
+        if (!components) return;
+
+        context[activeKey] = this;
+
+        if (debug) console.log("refreshing volume profile", components);
+
+        this._lastVolumeComponents = [...components];
+
+        // store all effects in an array to apply them all in one pass
+        // const effects: Array<Effect | Pass> = [];
+        this._effects.length = 0;
+
+        for (const component of components) {
+            //@ts-ignore
+            component.context = context;
+            if (component.apply) {
+                if (component.active) {
+                    if (!context.mainCameraComponent) {
+                        console.error("No camera in scene found or available yet - can not create postprocessing effects");
+                        return;
+                    }
+                    // apply or collect effects
+                    const res = component.apply();
+                    if (res === undefined || res === null) continue;
+                    if (Array.isArray(res)) {
+                        this._effects.push(...res);
+                    }
+                    else this._effects.push(res);
+                }
+            }
+            else {
+                if (component.active)
+                    showBalloonWarning("Volume component is not a VolumeComponent: " + component["__type"]);
+            }
+        }
+
+        this.applyEffects(context);
+
+    }
+
+
+    /** Build composer passes */
+    private applyEffects(context: Context) {
+
+        const effectsOrPasses = this._effects;
+        const camera = context.mainCameraComponent as Camera;
+        const renderer = context.renderer;
+
+        if (effectsOrPasses.length <= 0) return;
+
+        const cam = camera.cam;
+
+        // create composer and set active on context
+        if (!this._composer) {
+            // const hdrRenderTarget = new WebGLRenderTarget(window.innerWidth, window.innerHeight, { type: HalfFloatType });
+            this._composer = new EffectComposer(renderer, {
+                frameBufferType: HalfFloatType,
+                // multisampling: 2,
+            });
+        }
+        context.composer = this._composer;
+        const composer = context.composer;
+        composer.setMainCamera(cam);
+        composer.setRenderer(renderer);
+        for (const prev of composer.passes)
+            prev.dispose();
+        composer.removeAllPasses();
+        composer.addPass(new RenderPass(context.scene, cam));
+        
+
+        if (debug)
+            console.log("Set effects or passes", camera, effectsOrPasses, composer);
+
+
+        const effects: Array<Effect> = [];
+
+        for (const ef of effectsOrPasses) {
+            if (ef instanceof Effect)
+                effects.push(ef as Effect);
+            else if (ef instanceof Pass) {
+                const pass = new EffectPass(cam, ...effects);
+                pass.name = effects.map(e => e.constructor.name).join(", ");
+                pass.enabled = true;
+                composer.addPass(pass);
+                effects.length = 0;
+                composer.addPass(ef as Pass);
+            }
+        }
+
+        // create and apply uber pass
+        if (effects.length > 0) {
+            const pass = new EffectPass(cam, ...effects);
+            pass.enabled = true;
+            composer.addPass(pass);
+        }
+    }
+}
+

package/src/engine-components/postprocessing/Volume.ts

@@ -0,0 +1,194 @@
+import { Behaviour, GameObject } from "../Component";
+import { serializeable } from "../../engine/engine_serialization_decorator";
+import { getParam } from "../../engine/engine_utils";
+import { VolumeProfile } from "./VolumeProfile";
+import { EditorModification, IEditorModification as IEditorModificationReceiver } from "../../engine/engine_editor-sync";
+import { PostProcessingHandler } from "./PostProcessingHandler";
+import { PostProcessingEffect } from "./PostProcessingEffect";
+import { VolumeParameter } from "./VolumeParameter";
+import { getEditorModificationCache } from "../../engine/engine_editor-sync";
+import { isDevEnvironment } from "../../engine/debug";
+// import { CustomPostProcessingEffect } from "./CustomPostProcessingEffect";
+
+const debug = getParam("debugpost");
+
+export class Volume extends Behaviour implements IEditorModificationReceiver {
+
+    @serializeable(VolumeProfile)
+    sharedProfile?: VolumeProfile;
+
+    private _postprocessing: PostProcessingHandler = new PostProcessingHandler();
+    private _effects: PostProcessingEffect[] = [];
+
+    awake() {
+        // ensure the profile is initialized
+        this.sharedProfile?.init();
+
+        if (debug) {
+            console.log(this);
+            console.log("Press P to toggle post processing");
+            window.addEventListener("keydown", (e) => {
+                if (e.key === "p") {
+                    console.log("Toggle volume: " + this.name, !this.enabled);
+                    this.enabled = !this.enabled;
+                }
+            });
+        }
+    }
+
+    start() {
+        this.apply();
+        this.tryApplyCache();
+    }
+
+    onEnable() {
+        // We need to wait for the camera to be ready
+        if (this.__internalDidAwakeAndStart)
+            this.apply();
+    }
+
+    onDisable() {
+        this._postprocessing?.unapply(this.context);
+    }
+
+    onBeforeRender(): void {
+        if (!this.context.isInXR && this.context.composer && this.context.mainCamera) {
+            this.context.composer.setRenderer(this.context.renderer);
+            this.context.composer.setMainScene(this.context.scene);
+            this.context.composer.setMainCamera(this.context.mainCamera);
+        }
+    }
+
+    onDestroy(): void {
+        this._postprocessing?.dispose(this.context);
+    }
+
+    private _lastApplyTime?: number;
+    private _rapidApplyCount = 0;
+
+    private apply() {
+        if (debug) console.log("Apply PostProcessing", this);
+
+        if (isDevEnvironment()) {
+            if (this._lastApplyTime !== undefined && Date.now() - this._lastApplyTime < 100) {
+                this._rapidApplyCount++;
+                if (this._rapidApplyCount === 5)
+                    console.warn("Detected rapid post processing modifications - this might be a bug", this);
+            }
+            this._lastApplyTime = Date.now();
+        }
+
+
+        this.unapply();
+
+        this._effects.length = 0;
+        // get from profile
+        if (this.sharedProfile?.components) {
+            this._effects.push(...this.sharedProfile.components);
+        }
+        // get additional effects
+        const additionalComponents = this.gameObject.getComponentsInChildren(PostProcessingEffect);
+        if (debug && additionalComponents?.length)
+            console.log("Additional", additionalComponents);
+        if (additionalComponents) {
+            for (const comp of additionalComponents) {
+                if (comp.active) this._effects.push(comp);
+            }
+        }
+        if (this._effects.length > 0) {
+            this._postprocessing.apply(this.context, this._effects);
+        }
+    }
+
+    private unapply() {
+        this._postprocessing.unapply(this.context);
+    }
+
+
+    private tryApplyCache() {
+        const cache = getEditorModificationCache();
+        if (cache) {
+            for (const entry of cache.values()) this.onEditorModification(entry);
+        }
+    }
+
+
+    /** called from needle editor sync package if its active */
+    onEditorModification(modification: EditorModification): void | boolean | undefined {
+
+        if (modification.propertyName.startsWith("postprocessing.")) {
+            if (!this._effects?.length) return;
+            const path = modification.propertyName.split(".");
+            if (path.length === 3 || path.length === 4) {
+                const componentName = path[1];
+                const propertyName = path[2];
+                for (const comp of this._effects) {
+                    if (comp.typeName?.toLowerCase() === componentName.toLowerCase()) {
+
+                        if (propertyName === "active") {
+                            comp.active = modification.value;
+                            this.scheduleRecreate();
+                            return;
+                        }
+
+                        // cache the volume parameters
+                        if (!effectVolumeProperties.has(componentName)) {
+                            const volumeParameterKeys = new Array<string>();
+                            effectVolumeProperties.set(componentName, volumeParameterKeys);
+                            const keys = Object.keys(comp);
+                            for (const key of keys) {
+                                const prop = comp[key];
+                                if (prop instanceof VolumeParameter) {
+                                    volumeParameterKeys.push(key);
+                                }
+                            }
+                        }
+
+                        if (effectVolumeProperties.has(componentName)) {
+                            const paramName = propertyName.toLowerCase();
+                            const volumeParameterKeys = effectVolumeProperties.get(componentName)!;
+                            for (const key of volumeParameterKeys) {
+                                if (key.toLowerCase() === paramName) {
+                                    const prop = comp[key] as VolumeParameter;
+                                    if (prop instanceof VolumeParameter) {
+                                        const isActiveStateChange = path.length === 4 && path[3] === "active";
+                                        if (isActiveStateChange) {
+                                            prop.overrideState = modification.value;
+                                            this.scheduleRecreate();
+                                        }
+                                        else if (prop && prop.value !== undefined) {
+                                            prop.value = modification.value;
+                                        }
+                                    }
+                                    return;
+                                }
+                            }
+                        }
+
+                        console.warn("Unknown modification", propertyName);
+                        return;
+                    }
+                }
+            }
+            return true;
+        }
+
+        return false;
+    }
+
+    private _recreateId: number = -1;
+    private scheduleRecreate() {
+        // When the editor modifications come in with changed active effects we want/need to re-create the effects
+        // We defer it slightly because multiple active changes could be made and we dont want to recreate the full effect stack multiple times
+        const id = ++this._recreateId;
+        setTimeout(() => {
+            if (id !== this._recreateId) return;
+            this.onDisable();
+            this.onEnable();
+        }, 200);
+    }
+
+}
+
+/** cached VolumeParameter keys per object */
+const effectVolumeProperties: Map<string, string[]> = new Map<string, string[]>();

package/src/engine-components/postprocessing/VolumeParameter.ts

@@ -0,0 +1,85 @@
+import { serializable } from "../../engine/engine_serialization";
+
+export declare type VolumeParameterChangedEvent = (newValue: any, oldValue: any, parameter: VolumeParameter) => void;
+export declare type VolumeParameterValueProcessor = (value: any) => any;
+
+export class VolumeParameter {
+
+    @serializable()
+    get overrideState(): boolean {
+        return this._active;
+    }
+    set overrideState(val: boolean) {
+        if (this._active === val) return;
+        this._active = val;
+        const value = val ? this._lastActiveSetValue : this._defaultValue;
+        this.processValue(value, true);
+    }
+    private _active: boolean = false;
+
+
+
+    @serializable()
+    get value() {
+        return this._value;
+    }
+    set value(val: any) {
+        this.processValue(val, false);
+    }
+    private _value: any;
+    private _lastActiveSetValue?: any;
+
+
+    set defaultValue(val: any) {
+        this._defaultValue = val;
+    }
+    private _defaultValue: any = undefined;
+
+
+    /** enforce the value to be set and onValueChanged to be called if assigned */
+    __init() {
+        this.processValue(this._lastActiveSetValue, true);
+    }
+
+    /** called to modify a changing value before it is saved */
+    valueProcessor: VolumeParameterValueProcessor | undefined;
+    /** called when a value has changed (with the final value) */
+    onValueChanged: VolumeParameterChangedEvent | undefined;
+
+
+    private processValue(val: any, forceUpdate: boolean): any {
+        if (val === null || val === undefined) return;
+
+        if (!forceUpdate && this.testIfValueChanged(val) === false)
+            return;
+
+        const oldValue = this._value;
+
+        if (!this._active && this._defaultValue !== undefined) {
+            // when setting the default value we dont process them (default values are explicitly set from the effect that declares them 
+            // with the value that is expected to received when the parameter is disabled)
+            this._value = this._defaultValue;
+            val = this._defaultValue;
+        }
+        else {
+            this._lastActiveSetValue = val;
+            if (this._active && this.valueProcessor)
+                val = this.valueProcessor(val);
+            this._value = val;
+        }
+
+        if (this.onValueChanged) {
+            this.onValueChanged(val, oldValue, this);
+        }
+    }
+
+    private testIfValueChanged(newValue: any): boolean {
+
+        if (this._lastActiveSetValue === newValue)
+            return false;
+
+        // TODO: may need checks for colors or vectors (check by xyz,rgb because they might come in as anonymous objects via editor modifications)
+
+        return true;
+    }
+}

package/src/engine-components/postprocessing/VolumeProfile.ts

@@ -0,0 +1,40 @@
+import { serializeable } from "../../engine/engine_serialization_decorator";
+import { getParam } from "../../engine/engine_utils";
+import { PostProcessingEffect } from "./PostProcessingEffect";
+
+const debug = getParam("debugpost");
+
+const customEffects: { [name: string]: typeof PostProcessingEffect } = {};
+export function registerCustomEffectType(name: string, effect: typeof PostProcessingEffect) {
+    customEffects[name] = effect;
+}
+
+// resolve the types:
+function resolveComponentType(data: { __type: string }) {
+    if (data.__type in customEffects)
+        return customEffects[data.__type];
+    // if ("mode" in data) return ToneMapping;
+    // if ("postExposure" in data) return ColorAdjustments;
+    // switch (data.__type) {
+    //     // case "Bloom": return Bloom;
+    //     // case "DepthOfField": return DepthOfField;
+    //     // case "Vignette": return Vignette
+    //     // case "ColorAdjustments": return ColorAdjustments;
+    //     // case "Tonemapping": return ToneMapping;
+    // }
+    if (debug && data.__type)
+        console.warn("Unknown postprocessing type", data.__type, data)
+    return PostProcessingEffect;
+}
+
+export class VolumeProfile {
+
+    @serializeable([d => resolveComponentType(d), PostProcessingEffect])
+    components?: PostProcessingEffect[];
+
+    /** call init on all components */
+    init() {
+        this.components?.forEach(c => c.init());
+    }
+}
+

package/src/engine-components/timeline/TimelineTracks.ts

@@ -658,25 +658,24 @@ export class AudioTrackHandler extends TrackHandler {
 
     private handleAudioLoading(model: Models.ClipModel, audio: THREE.Audio): Promise<AudioBuffer | null> | null {
         if (!this._audioLoader) {
-            const path = this.getAudioFilePath(model.asset.clip);
             this._audioLoader = new THREE.AudioLoader();
-            // TODO: maybe we should cache the loaders / buffers here by path
-            if (debug) console.warn("LOAD audio track", path, this.director.sourceId);
-            const loadingPromise = new Promise<AudioBuffer | null>((resolve, _reject) => {
-                this._audioLoader!.load(path,
-                    buffer => {
-                        audio.setBuffer(buffer);
-                        resolve(buffer);
-                    },
-                    undefined,
-                    err => {
-                        console.error("Error loading audio", err);
-                        resolve(null);
-                    });
-            });
-            return loadingPromise;
         }
-        return null;
+        // TODO: maybe we should cache the loaders / buffers here by path
+        const path = this.getAudioFilePath(model.asset.clip);
+        if (debug) console.warn("LOAD audio track", path, this.director.sourceId);
+        const loadingPromise = new Promise<AudioBuffer | null>((resolve, _reject) => {
+            this._audioLoader!.load(path,
+                buffer => {
+                    audio.setBuffer(buffer);
+                    resolve(buffer);
+                },
+                undefined,
+                err => {
+                    console.error("Error loading audio", err);
+                    resolve(null);
+                });
+        });
+        return loadingPromise;
     }
 }