@needle-tools/engine 4.6.0 → 4.6.1-next.20d54cd

package/src/engine-components/postprocessing/PostProcessingHandler.ts

@@ -1,24 +1,26 @@
 import type { Effect, EffectComposer, Pass, ToneMappingEffect as _TonemappingEffect } from "postprocessing";
-import { HalfFloatType, NoToneMapping } from "three";
+import { Camera as Camera3, DepthTexture, HalfFloatType, LinearFilter, NoToneMapping, Scene, Texture, ToneMapping, WebGLRenderTarget } from "three";
 
 import { showBalloonWarning } from "../../engine/debug/index.js";
 // import { internal_SetSharpeningEffectModule } from "./Effects/Sharpening.js";
 import { MODULES } from "../../engine/engine_modules.js";
 import { Context } from "../../engine/engine_setup.js";
-import type { Constructor } from "../../engine/engine_types.js";
-import { DeviceUtilities, getParam } from "../../engine/engine_utils.js";
+import { Graphics } from "../../engine/engine_three_utils.js";
+import { getParam } from "../../engine/engine_utils.js";
 import { Camera } from "../Camera.js";
+import { threeToneMappingToEffectMode } from "./Effects/Tonemapping.utils.js";
 import { PostProcessingEffect, PostProcessingEffectContext } from "./PostProcessingEffect.js";
+import { orderEffects, PostprocessingEffectData, PostProcessingEffectOrder } from "./utils.js";
 
 declare const NEEDLE_USE_POSTPROCESSING: boolean;
 globalThis["NEEDLE_USE_POSTPROCESSING"] = globalThis["NEEDLE_USE_POSTPROCESSING"] !== undefined ? globalThis["NEEDLE_USE_POSTPROCESSING"] : true;
 
 
 const debug = getParam("debugpost");
-const dontMergePasses = getParam("debugpostpasses");
 
 const activeKey = Symbol("needle:postprocessing-handler");
 const autoclearSetting = Symbol("needle:previous-autoclear-state");
+const previousToneMapping = Symbol("needle:previous-tone-mapping");
 
 /**
  * PostProcessingHandler is responsible for applying post processing effects to the scene. It is internally used by the {@link Volume} component
@@ -27,7 +29,15 @@ export class PostProcessingHandler {
 
     private _composer: EffectComposer | null = null;
     private _lastVolumeComponents?: PostProcessingEffect[];
-    private _effects: Array<Effect | Pass> = [];
+    private readonly _effects: Array<PostprocessingEffectData> = [];
+
+    /**
+     * Returns true if a specific effect is currently active in the post processing stack.
+     */
+    getEffectIsActive(effect: Effect): boolean {
+        if (!effect) return false;
+        return this._isActive && this._effects.some(e => e.effect === effect);
+    }
 
     get isActive() {
         return this._isActive;
@@ -44,7 +54,7 @@ export class PostProcessingHandler {
         this.context = context;
     }
 
-    apply(components: PostProcessingEffect[]) : Promise<void> {
+    apply(components: PostProcessingEffect[]): Promise<void> {
         if ("env" in import.meta && import.meta.env.VITE_NEEDLE_USE_POSTPROCESSING === "false") {
             if (debug) console.warn("Postprocessing is disabled via vite env setting");
             else console.debug("Postprocessing is disabled via vite env setting");
@@ -60,7 +70,7 @@ export class PostProcessingHandler {
         return this.onApply(this.context, components);
     }
 
-    unapply() {
+    unapply(dispose: boolean = true) {
         if (debug) console.log("Unapplying postprocessing effects");
         this._isActive = false;
         if (this._lastVolumeComponents) {
@@ -73,21 +83,29 @@ export class PostProcessingHandler {
         const active = context[activeKey] as PostProcessingHandler | null;
         if (active === this) {
             delete context[activeKey];
+
+            // Restore the auto clear setting
+            if (typeof context.renderer[autoclearSetting] === "boolean") {
+                context.renderer.autoClear = context.renderer[autoclearSetting];
+            }
+            if (typeof context.renderer[previousToneMapping] === "number") {
+                context.renderer.toneMapping = context.renderer[previousToneMapping] as ToneMapping;
+            }
         }
+
+        this._composer?.removeAllPasses();
+        if (dispose) this._composer?.dispose();
+
         if (context.composer === this._composer) {
-            context.composer?.dispose();
             context.composer = null;
         }
-        if (typeof context.renderer[autoclearSetting] === "boolean") {
-            context.renderer.autoClear = context.renderer[autoclearSetting];
-        }
     }
 
     dispose() {
-        this.unapply();
+        this.unapply(true);
 
         for (const effect of this._effects) {
-            effect.dispose();
+            effect.effect.dispose();
         }
         this._effects.length = 0;
         this._composer = null;
@@ -105,6 +123,7 @@ export class PostProcessingHandler {
             // import("./Effects/Sharpening.effect")
         ]);
 
+
         // try {
         //     internal_SetSharpeningEffectModule(modules[2]);
         // }
@@ -141,10 +160,22 @@ export class PostProcessingHandler {
                     // apply or collect effects
                     const res = component.apply(ctx);
                     if (!res) continue;
+
+
                     if (Array.isArray(res)) {
-                        this._effects.push(...res);
+                        for (const effect of res) {
+                            this._effects.push({
+                                effect,
+                                priority: component.order
+                            });
+                        }
+                    }
+                    else {
+                        this._effects.push({
+                            effect: res,
+                            priority: component.order
+                        });
                     }
-                    else this._effects.push(res);
                 }
             }
             else {
@@ -153,24 +184,40 @@ export class PostProcessingHandler {
             }
         }
 
-        // Ensure that we have a tonemapping effect if the renderer is set to use a tone mapping
-        if (this.context.renderer.toneMapping != NoToneMapping) {
-            if (!this._effects.find(e => e instanceof MODULES.POSTPROCESSING.MODULE.ToneMappingEffect)) {
-                const tonemapping = new MODULES.POSTPROCESSING.MODULE.ToneMappingEffect();
-                this._effects.push(tonemapping);
-            }
-        }
-
         this.applyEffects(context);
     }
 
+    private _anyPassHasDepth = false;
+    private _anyPassHasNormal = false;
+    private _hasSmaaEffect = false;
+
+    get anyPassHasDepth() { return this._anyPassHasDepth; }
+    get anyPassHasNormal() { return this._anyPassHasNormal; }
+    get hasSmaaEffect() { return this._hasSmaaEffect; }
+
+
+
+    private _customInputBuffer: WebGLRenderTarget<Texture> | null = null;
+    private _customInputBufferId = 0;
+    private _multisampling: number = 0;
+    set multisampling(value: number) {
+        this._multisampling = value;
+    }
+    get multisampling() {
+        return this._multisampling;
+    }
+
 
     /** Build composer passes */
     private applyEffects(context: Context) {
+        // Reset state
+        this._anyPassHasDepth = false;
+        this._anyPassHasNormal = false;
+        this._hasSmaaEffect = false;
 
-        const effectsOrPasses = this._effects;
-        
-        if (effectsOrPasses.length <= 0) return;
+        if (this._effects.length <= 0) {
+            return;
+        }
 
         const camera = context.mainCameraComponent as Camera;
         const renderer = context.renderer;
@@ -180,8 +227,27 @@ export class PostProcessingHandler {
         // Store the auto clear setting because the postprocessing composer just disables it
         // and when we disable postprocessing we want to restore the original setting
         // https://github.com/pmndrs/postprocessing/blob/271944b74b543a5b743a62803a167b60cc6bb4ee/src/core/EffectComposer.js#L230C12-L230C12
+        // First we need to get the previously set autoClear setting, if it exists
+        if (typeof renderer[autoclearSetting] === "boolean") {
+            renderer.autoClear = renderer[autoclearSetting];
+        }
         renderer[autoclearSetting] = renderer.autoClear;
 
+        if (typeof renderer[previousToneMapping] === "number") {
+            renderer.toneMapping = renderer[previousToneMapping] as ToneMapping;
+        }
+        renderer[previousToneMapping] = renderer.toneMapping;
+
+        // Ensure that we have a tonemapping effect if the renderer is set to use a tone mapping
+        if (renderer.toneMapping != NoToneMapping) {
+            if (!this._effects.find(e => e instanceof MODULES.POSTPROCESSING.MODULE.ToneMappingEffect)) {
+                const tonemapping = new MODULES.POSTPROCESSING.MODULE.ToneMappingEffect();
+                tonemapping.name = `ToneMapping (${renderer.toneMapping})`;
+                tonemapping.mode = threeToneMappingToEffectMode(renderer.toneMapping);
+                this._effects.push({ effect: tonemapping, priority: PostProcessingEffectOrder.ToneMapping });
+            }
+        }
+
         // create composer and set active on context
         if (!this._composer) {
             // const hdrRenderTarget = new WebGLRenderTarget(window.innerWidth, window.innerHeight, { type: HalfFloatType });
@@ -194,11 +260,14 @@ export class PostProcessingHandler {
         if (context.composer && context.composer !== this._composer) {
             console.warn("There's already an active EffectComposer in your scene: replacing it with a new one. This might cause unexpected behaviour. Make sure to only use one PostprocessingManager/Volume in your scene.");
         }
+
         context.composer = this._composer;
         const composer = context.composer;
         composer.setMainCamera(cam);
         composer.setRenderer(renderer);
         composer.setMainScene(scene);
+        composer.autoRenderToScreen = true; // Must be enabled because it might be disabled during addPass by the composer itself (depending on the effect's settings or order)
+        composer.multisampling = 0; // Disable multisampling by default
 
         for (const prev of composer.passes)
             prev.dispose();
@@ -206,68 +275,197 @@ export class PostProcessingHandler {
 
         // Render to screen pass
         const screenpass = new MODULES.POSTPROCESSING.MODULE.RenderPass(scene, cam);
-        screenpass.name = "Render To Screen";
+        screenpass.name = "RenderPass";
         screenpass.mainScene = scene;
         composer.addPass(screenpass);
 
-        const automaticEffectsOrdering = true;
-        if (automaticEffectsOrdering && !dontMergePasses) {
-            try {
-                this.orderEffects();
+        const screenPassRender = screenpass.render;
+        this._customInputBuffer?.dispose();
+        this._customInputBuffer = null;
+        screenpass.render = (renderer, inputBuffer, outputBuffer, deltaTime, stencilTest) => {
+            if (!inputBuffer) return;
 
-                const effects: Array<Effect> = [];
+            // screenPassRender.call(screenpass, renderer, inputBuffer, outputBuffer, deltaTime, stencilTest);
+            // return;
 
-                for (const ef of effectsOrPasses) {
-                    if (ef instanceof MODULES.POSTPROCESSING.MODULE.Effect)
-                        effects.push(ef as Effect);
-                    else if (ef instanceof MODULES.POSTPROCESSING.MODULE.Pass) {
-                        composer.addPass(ef as Pass);
-                    }
-                    else {
-                        // seems some effects are not correctly typed, but three can deal with them,
-                        // so we might need to just pass them through
-                        composer.addPass(ef);
-                    }
-                }
+            // Make sure multisampling is disabled on the composer buffers. Technically a user could still set multisampling directly on the composer so this is to override that and make sure these textures do NOT use multisampling
+            inputBuffer.samples = 0;
+            if (outputBuffer) {
+                outputBuffer.samples = 0;
+            }
 
-                // create and apply uber pass
-                if (effects.length > 0) {
-                    const pass = new MODULES.POSTPROCESSING.MODULE.EffectPass(cam, ...effects);
-                    pass.name = effects.map(e => e.name).join(" ");
-                    pass.mainScene = scene;
-                    pass.enabled = true;
-                    composer.addPass(pass);
+            // Make sure the input buffer is a WebGLRenderTarget with the correct settings
+            if (!this._customInputBuffer
+                || this._customInputBuffer.width !== inputBuffer.width
+                || this._customInputBuffer.height !== inputBuffer.height
+                || this._customInputBuffer.samples !== this._multisampling
+                || this._customInputBuffer.texture.format !== inputBuffer.texture.format
+                || this._customInputBuffer.texture.type !== HalfFloatType
+            ) {
+                this._customInputBuffer?.dispose();
+
+                this._customInputBuffer = new WebGLRenderTarget(inputBuffer.width, inputBuffer.height, {
+                    format: inputBuffer.texture.format,
+                    type: HalfFloatType,
+                    depthBuffer: inputBuffer.depthBuffer,
+                    depthTexture: inputBuffer.depthTexture
+                        ? new DepthTexture(inputBuffer.width, inputBuffer.height)
+                        : undefined,
+                    stencilBuffer: inputBuffer.stencilBuffer,
+                    samples: Math.max(0, this._multisampling),
+                    minFilter: inputBuffer.texture.minFilter ?? LinearFilter,
+                    magFilter: inputBuffer.texture.magFilter ?? LinearFilter,
+                    generateMipmaps: inputBuffer.texture.generateMipmaps,
+                });
+                this._customInputBufferId++;
+                this._customInputBuffer.texture.name = `CustomInputBuffer-${this._customInputBufferId}`;
+                if (this._customInputBuffer.depthTexture && inputBuffer.depthTexture) {
+                    this._customInputBuffer.depthTexture.format = inputBuffer.depthTexture.format;
+                    this._customInputBuffer.depthTexture.type = inputBuffer.depthTexture.type;
                 }
+                // https://github.com/pmndrs/postprocessing/blob/ad338df710ef41fee4e5d10ad2c2c299030d46ef/src/core/EffectComposer.js#L366
+                if (this._customInputBuffer.samples > 0)
+                    (this._customInputBuffer as any).ignoreDepthForMultisampleCopy = false;
+
+                if (debug) console.warn(`[PostProcessing] Input buffer created with size ${this._customInputBuffer.width}x${this._customInputBuffer.height} and samples ${this._customInputBuffer.samples}`);
             }
-            catch (e) {
-                console.error("Error while applying postprocessing effects", e);
-                composer.removeAllPasses();
+            // Calling the original render function with the input buffer
+            screenPassRender.call(screenpass, renderer, this._customInputBuffer, outputBuffer, deltaTime, stencilTest);
+            // Blit the resulting buffer to the input buffer passed in by the composer so it's used for subsequent effects
+            Graphics.blit(this._customInputBuffer.texture, inputBuffer, {
+                renderer,
+                depthTexture: this._customInputBuffer.depthTexture,
+                depthWrite: true,
+                depthTest: true,
+            });
+        };
+
+
+        try {
+            orderEffects(this._effects);
+
+            let foundTonemappingEffect = false;
+            let activeTonemappingEffect: _TonemappingEffect | null = null;
+            for (let i = this._effects.length - 1; i >= 0; i--) {
+                const ef = this._effects[i].effect;
+                if (ef instanceof MODULES.POSTPROCESSING.MODULE.ToneMappingEffect) {
+                    // If we already have a tonemapping effect, we can skip this one
+                    if (foundTonemappingEffect) {
+                        if (debug) console.warn(`[PostProcessing] Found multiple tonemapping effects in the scene: ${ef.name} and ${activeTonemappingEffect?.name}. Only the last one added will be used.`);
+                        this._effects.splice(i, 1);
+                        continue;
+                    }
+                    activeTonemappingEffect = ef;
+                    foundTonemappingEffect = true;
+                }
             }
-        }
-        else {
-            // we still want to sort passes, but we do not want to merge them for debugging
-            if (automaticEffectsOrdering)
-                this.orderEffects();
 
-            for (const ef of effectsOrPasses) {
+            const effectsToMerge: Array<Effect> = [];
+            let hasConvolutionEffectInArray = false;
+
+            for (const entry of this._effects) {
+                const ef = entry.effect;
+
+                if (ef instanceof MODULES.POSTPROCESSING.MODULE.SMAAEffect) {
+                    this._hasSmaaEffect = true;
+                }
+                else if (ef instanceof MODULES.POSTPROCESSING.MODULE.NormalPass) {
+                    this._anyPassHasNormal = true;
+                }
+
+
+                // There can be only one tonemapping effect in the scene, so we can skip all others
+                if (ef instanceof MODULES.POSTPROCESSING.MODULE.ToneMappingEffect && activeTonemappingEffect !== ef) {
+                    // If we already have a tonemapping effect, we can skip this one
+                    continue;
+                }
+
+                // We can also not merge multiple effects of the same type in one pass
+                // So we first need to create a new pass with whatever effects we have so far
+                // TODO: this seems to work fine for some effects (like ColorAdjustments) and only caused issues with multiple Tonemapping effects so far which is handled above
+                // const constructor = ef.constructor as Constructor<Effect | Pass>;
+                // if(effectsToMerge.find(e => e.constructor === constructor)) {
+                //     // this.createPassForMergeableEffects(effectsToMerge, composer, cam, scene);
+                // }
+
                 if (ef instanceof MODULES.POSTPROCESSING.MODULE.Effect) {
-                    const pass = new MODULES.POSTPROCESSING.MODULE.EffectPass(cam, ef as Effect);
-                    pass.name = ef.name;
-                    composer.addPass(pass);
+                    const attributes = ef.getAttributes();
+                    const convolution = MODULES.POSTPROCESSING.MODULE.EffectAttribute.CONVOLUTION;
+                    if (attributes & convolution) {
+                        if (debug) console.log("[PostProcessing] Convolution effect: " + ef.name);
+                        if (hasConvolutionEffectInArray) {
+                            if (debug) console.log("[PostProcessing] Merging effects with convolution", effectsToMerge.map(e => e.name).join(", "));
+                            this.createPassForMergeableEffects(effectsToMerge, composer, cam, scene);
+                        }
+                        hasConvolutionEffectInArray = true;
+                    }
+                    // Otherwise we can merge it
+                    effectsToMerge.push(ef as Effect);
                 }
-                else if (ef instanceof MODULES.POSTPROCESSING.MODULE.Pass)
+                else if (ef instanceof MODULES.POSTPROCESSING.MODULE.Pass) {
+                    hasConvolutionEffectInArray = false;
+                    this.createPassForMergeableEffects(effectsToMerge, composer, cam, scene);
+                    ef.renderToScreen = false;
                     composer.addPass(ef as Pass);
-                else
+                }
+                else {
                     // seems some effects are not correctly typed, but three can deal with them,
-                    // so we just pass them through
+                    // so we might need to just pass them through
+                    hasConvolutionEffectInArray = false;
+                    this.createPassForMergeableEffects(effectsToMerge, composer, cam, scene);
                     composer.addPass(ef);
+                }
+            }
+
+            this.createPassForMergeableEffects(effectsToMerge, composer, cam, scene);
+        }
+        catch (e) {
+            console.error("Error while applying postprocessing effects", e);
+            composer.removeAllPasses();
+        }
+
+        // The last pass is the one that renders to the screen, so we need to set the gamma correction for it (and enable it for all others)
+        for (let i = 0; i < composer.passes.length; i++) {
+            const pass = composer.passes[i];
+            const isLast = i === composer.passes.length - 1;
+            if ((pass as any)?.configuration !== undefined) {
+                (pass as any).configuration.gammaCorrection = isLast;
             }
+            else if ("autosetGamma" in pass) {
+                // Some effects have a autosetGamma property that we can use to set the gamma correction
+                pass.autosetGamma = isLast;
+            }
+
+            this._anyPassHasDepth ||= pass.needsDepthTexture;
         }
 
-        if (debug) {
+        // DEBUG LAND BELOW
+        if (debug) console.log("[PostProcessing] Passes →", [...composer.passes], "\n---------------------------------\n• " + composer.passes.map(i => i.name).join("\n• ") + "\n");
+        if (debug) this._onCreateEffectsDebug(this._composer!, cam);
+    }
+
+
 
-            console.log("[PostProcessing] Passes →", composer.passes);
+    /** Should be called before `composer.addPass()` to create an effect pass with all previously collected effects that can be merged up to that point */
+    private createPassForMergeableEffects(effects: Array<Effect>, composer: EffectComposer, camera: Camera3, scene: Scene) {
+        if (effects.length > 0) {
+            const pass = new MODULES.POSTPROCESSING.MODULE.EffectPass(camera, ...effects);
+            pass.name = effects.map(e => e.name).join(", ");
+            pass.mainScene = scene;
+            pass.enabled = true;
+            pass.renderToScreen = false;
+            composer.addPass(pass);
+            effects.length = 0; // Clear effects after adding them to the pass
+        }
+    }
+
+
+
+    private _menuEntry: HTMLSelectElement | null = null;
+    private _passIndices: number[] | null = null;
 
+    private _onCreateEffectsDebug(composer: EffectComposer, cam: Camera3) {
+        if (debug === "passes") {
             // DepthEffect for debugging purposes, disabled by default, can be selected in the debug pass select
             const depthEffect = new MODULES.POSTPROCESSING.MODULE.DepthEffect({
                 blendFunction: MODULES.POSTPROCESSING.MODULE.BlendFunction.NORMAL,
@@ -303,7 +501,7 @@ export class PostProcessingHandler {
             if (menu && this._passIndices === null) {
                 if (this._menuEntry)
                     this._menuEntry.remove();
-                
+
                 const select = document.createElement("select");
                 select.multiple = true;
                 const defaultOpt = document.createElement("option");
@@ -327,73 +525,11 @@ export class PostProcessingHandler {
                     else {
                         this._passIndices = indices;
                     }
-
-                    this.applyEffects(context);
+                    this.applyEffects(this.context);
                 });
             }
         }
     }
 
-    private _menuEntry: HTMLSelectElement | null = null;
-    private _passIndices: number[] | null = null;
-
-    private orderEffects() {
-        if (debug) console.log("Before ordering effects", [...this._effects]);
-
-
-
-        // Order of effects for correct results.
-        // Aligned with https://github.com/pmndrs/postprocessing/wiki/Effect-Merging#effect-execution-order
-        // We can not put this into global scope because then the module might not yet be initialized
-        effectsOrder ??= [
-            MODULES.POSTPROCESSING.MODULE.NormalPass,
-            MODULES.POSTPROCESSING.MODULE.DepthDownsamplingPass,
-            MODULES.POSTPROCESSING.MODULE.SMAAEffect,
-            MODULES.POSTPROCESSING.MODULE.SSAOEffect,
-            MODULES.POSTPROCESSING_AO.MODULE.N8AOPostPass,
-            MODULES.POSTPROCESSING.MODULE.TiltShiftEffect,
-            MODULES.POSTPROCESSING.MODULE.DepthOfFieldEffect,
-            MODULES.POSTPROCESSING.MODULE.ChromaticAberrationEffect,
-            MODULES.POSTPROCESSING.MODULE.BloomEffect,
-            MODULES.POSTPROCESSING.MODULE.SelectiveBloomEffect,
-            MODULES.POSTPROCESSING.MODULE.VignetteEffect,
-            MODULES.POSTPROCESSING.MODULE.PixelationEffect,
-            MODULES.POSTPROCESSING.MODULE.ToneMappingEffect,
-            MODULES.POSTPROCESSING.MODULE.HueSaturationEffect,
-            MODULES.POSTPROCESSING.MODULE.BrightnessContrastEffect,
-            // __SHARPENING_MODULE._SharpeningEffect,
-        ];
-
-
-        // TODO: enforce correct order of effects (e.g. DOF before Bloom)
-        const effects = this._effects;
-        effects.sort((a, b) => {
-            // we use find index here because sometimes constructor names are prefixed with `_`
-            // TODO: find a more robust solution that isnt name based (not sure if that exists tho... maybe we must give effect TYPES some priority/index)
-            const aidx = effectsOrder!.findIndex(e => a.constructor.name.endsWith(e.name));
-            const bidx = effectsOrder!.findIndex(e => b.constructor.name.endsWith(e.name));
-            // Unknown effects should be rendered first
-            if (aidx < 0) {
-                if (debug) console.warn("Unknown effect found: ", a.constructor.name);
-                return -1;
-            }
-            else if (bidx < 0) {
-                if (debug) console.warn("Unknown effect found: ", b.constructor.name);
-                return 1;
-            }
-            if (aidx < 0) return 1;
-            if (bidx < 0) return -1;
-            return aidx - bidx;
-        });
-        if (debug) console.log("After ordering effects", [...this._effects]);
-        for (let i = 0; i < effects.length; i++) {
-            const effect = effects[i] as any;
-            if (effect?.configuration?.gammaCorrection !== undefined) {
-                const isLast = i === effects.length - 1;
-                effect.configuration.gammaCorrection = isLast;
-            }
-        }
-    }
 }
 
-let effectsOrder: Array<Constructor<Effect | Pass>> | null = null;