@babylonjs/smart-filters 0.1.0-alpha

package/src/runtime/renderTargetGenerator.ts

@@ -0,0 +1,204 @@
+import type { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture";
+import type { Nullable } from "@babylonjs/core/types";
+import { ThinRenderTargetTexture } from "@babylonjs/core/Materials/Textures/thinRenderTargetTexture.js";
+
+import type { BaseBlock } from "../blocks/baseBlock";
+import type { InitializationData, SmartFilter } from "../smartFilter";
+import type { InternalSmartFilterRuntime } from "./smartFilterRuntime";
+import { ShaderBlock } from "../blocks/shaderBlock.js";
+import { createStrongRef } from "./strongRef.js";
+import { ConnectionPointType } from "../connection/connectionPointType.js";
+
+/**
+ * @internal
+ */
+interface RefCountedTexture {
+    /**
+     * The texture.
+     */
+    texture: ThinTexture;
+
+    /**
+     * The reference count.
+     */
+    refCount: number;
+}
+
+/**
+ * The render target generator is responsible for creating and assigning render targets to ShaderBlocks.
+ */
+export class RenderTargetGenerator {
+    private _optimize: boolean;
+    private _renderTargetPool: Map<number, Set<RefCountedTexture>>;
+    private _numTargetsCreated;
+
+    /**
+     * Creates a new render target generator.
+     * @param optimize - If true, the render target generator will try to reuse render targets as much as possible.
+     */
+    constructor(optimize = true) {
+        this._optimize = optimize;
+        this._renderTargetPool = new Map();
+        this._numTargetsCreated = 0;
+    }
+
+    /**
+     * Returns the number of render targets created by the process
+     */
+    public get numTargetsCreated() {
+        return this._numTargetsCreated;
+    }
+
+    /**
+     * Sets the output textures for the ShaderBlocks of the smart filter.
+     * @param smart - The smart filter to generate the render targets for.
+     * @param initializationData - The initialization data to use.
+     */
+    public setOutputTextures(smart: SmartFilter, initializationData: InitializationData) {
+        smart.output.ownerBlock.visit(
+            initializationData,
+            (block: BaseBlock, initializationData: InitializationData) => {
+                if (!(block instanceof ShaderBlock)) {
+                    return;
+                }
+
+                let refCountedTexture: Nullable<RefCountedTexture> = null;
+
+                // We assign a texture to the output of the block only if this is not the last block in the chain,
+                // i.e. not the block connected to the smart output block (in which case the output of the block is to the canvas and not a texture).
+                if (!block.output.endpoints.some((cp) => cp.ownerBlock === smart.output.ownerBlock)) {
+                    refCountedTexture = this._getTexture(initializationData.runtime, block.textureRatio);
+
+                    if (!block.output.runtimeData) {
+                        const runtimeOutput = createStrongRef(refCountedTexture.texture);
+                        block.output.runtimeData = runtimeOutput;
+                    } else {
+                        block.output.runtimeData.value = refCountedTexture.texture;
+                    }
+                }
+
+                if (this._optimize) {
+                    if (refCountedTexture !== null) {
+                        for (const output of block.outputs) {
+                            output.endpoints.forEach((endpoint) => {
+                                if (endpoint.connectedTo) {
+                                    refCountedTexture!.refCount++;
+                                }
+                            });
+                        }
+                    }
+
+                    for (const input of block.inputs) {
+                        if (!input.connectedTo || input.connectedTo.type !== ConnectionPointType.Texture) {
+                            continue;
+                        }
+                        const connectedBlock = input.connectedTo.ownerBlock;
+                        if (connectedBlock instanceof ShaderBlock && connectedBlock.output.runtimeData) {
+                            this._releaseTexture(connectedBlock.output.runtimeData.value, connectedBlock.textureRatio);
+                        }
+                    }
+                }
+            }
+        );
+        this._renderTargetPool.clear();
+    }
+
+    private _findAvailableTexture(ratio: number): Nullable<RefCountedTexture> {
+        const refCountedTextures = this._renderTargetPool.get(ratio);
+        if (!refCountedTextures) {
+            return null;
+        }
+
+        for (const refCountedTexture of refCountedTextures) {
+            if (refCountedTexture.refCount === 0) {
+                return refCountedTexture;
+            }
+        }
+
+        return null;
+    }
+
+    private _getTexture(runtime: InternalSmartFilterRuntime, ratio: number): RefCountedTexture {
+        if (!this._optimize) {
+            this._numTargetsCreated++;
+            return {
+                texture: this._createTexture(runtime, ratio),
+                refCount: 0,
+            };
+        }
+
+        let refCountedTextures = this._renderTargetPool.get(ratio);
+        if (!refCountedTextures) {
+            refCountedTextures = new Set();
+            this._renderTargetPool.set(ratio, refCountedTextures);
+        }
+
+        let refCountedTexture = this._findAvailableTexture(ratio);
+        if (!refCountedTexture) {
+            refCountedTexture = {
+                texture: this._createTexture(runtime, ratio),
+                refCount: 0,
+            };
+            refCountedTextures.add(refCountedTexture);
+            this._numTargetsCreated++;
+        }
+
+        return refCountedTexture;
+    }
+
+    private _releaseTexture(texture: ThinTexture, ratio: number) {
+        if (!this._optimize) {
+            return;
+        }
+
+        const refCountedTextures = this._renderTargetPool.get(ratio);
+        if (!refCountedTextures) {
+            throw new Error(`_releaseTextureToPool: Trying to add a texture to a non existing pool ${ratio}!`);
+        }
+
+        for (const refCountedTexture of refCountedTextures) {
+            if (refCountedTexture.texture === texture) {
+                refCountedTexture.refCount--;
+                return;
+            }
+        }
+
+        throw new Error(`_releaseTextureToPool: Can't find the texture in the pool ${ratio}!`);
+    }
+
+    /**
+     * Creates an offscreen texture to hold on the result of the block rendering.
+     * @param runtime - The current runtime we create the texture for
+     * @param ratio - The ratio of the texture to create compared to the final output
+     * @returns The render target texture
+     */
+    private _createTexture(runtime: InternalSmartFilterRuntime, ratio: number): ThinRenderTargetTexture {
+        const engine = runtime.engine;
+
+        // We are only rendering full screen post process without depth or stencil information
+        const setup = {
+            generateDepthBuffer: false,
+            generateStencilBuffer: false,
+            generateMipMaps: false,
+            samplingMode: 2, // Babylon Constants.TEXTURE_LINEAR_LINEAR,
+        };
+
+        // The size of the output is by default the current rendering size of the engine
+        const width = engine.getRenderWidth(true);
+        const height = engine.getRenderHeight(true);
+        const size = {
+            width: Math.floor(width * ratio),
+            height: Math.floor(height * ratio),
+        };
+
+        // Creates frame buffers for effects
+        const finalRenderTarget = new ThinRenderTargetTexture(engine, size, setup);
+        runtime.registerResource(finalRenderTarget);
+
+        // Babylon Constants.TEXTURE_CLAMP_ADDRESSMODE; NPOT Friendly
+        finalRenderTarget.wrapU = 0;
+        finalRenderTarget.wrapV = 0;
+
+        return finalRenderTarget;
+    }
+}

package/src/runtime/shaderRuntime.ts

@@ -0,0 +1,155 @@
+import type { EffectRenderer } from "@babylonjs/core/Materials/effectRenderer";
+import type { RenderTargetWrapper } from "@babylonjs/core/Engines/renderTargetWrapper";
+import type { AbstractEngine } from "@babylonjs/core/Engines/abstractEngine";
+import type { Effect } from "@babylonjs/core/Materials/effect";
+import { EffectWrapper } from "@babylonjs/core/Materials/effectRenderer.js";
+
+import type { IDisposable } from "../IDisposable";
+import type { ShaderProgram } from "../utils/shaderCodeUtils";
+import { createStrongRef, type StrongRef } from "./strongRef.js";
+import type { IDisableableBlock } from "../blocks/disableableBlock";
+import { decorateSymbol, getShaderCreateOptions } from "../utils/shaderCodeUtils.js";
+
+/**
+ * The shader bindings for a ShaderBlock that can't be disabled.
+ */
+export abstract class Binding {
+    /**
+     * Binds all the required data to the shader when rendering.
+     * @param effect - defines the effect to bind the data to
+     * @param width - defines the width of the output
+     * @param height - defines the height of the output
+     */
+    public abstract bind(effect: Effect, width?: number, height?: number): void;
+
+    private _remappedShaderVariables: { [key: string]: string } = {};
+
+    /**
+     * Gets the remapped shader variable name.
+     * @param variableName - The variable name
+     * @returns The remapped variable name
+     */
+    public getRemappedName(variableName: string) {
+        variableName = decorateSymbol(variableName);
+        return this._remappedShaderVariables[variableName] ?? variableName;
+    }
+
+    /**
+     * Sets the remapped shader variables.
+     * @param variableName - defines the variable name to remap
+     * @param remappedName - defines the remapped name
+     */
+    public addShaderVariableRemapping(variableName: string, remappedName: string) {
+        this._remappedShaderVariables[variableName] = remappedName;
+    }
+}
+
+/**
+ * The shader bindings for a disableable ShaderBlock.
+ */
+export abstract class ShaderBinding extends Binding {
+    private _disabled: StrongRef<boolean>;
+
+    /**
+     * Construct a ShaderBinding instance.
+     * @param parentBlock - The parent block
+     */
+    constructor(parentBlock: IDisableableBlock) {
+        super();
+        this._disabled = parentBlock.disabled?.runtimeData || createStrongRef(false);
+    }
+
+    /**
+     * Binds all the required data to the shader when rendering.
+     * @param effect - defines the effect to bind the data to
+     * @param _width - defines the width of the output
+     * @param _height - defines the height of the output
+     */
+    public bind(effect: Effect, _width?: number, _height?: number): void {
+        effect.setBool(this.getRemappedName("disabled"), this._disabled.value);
+    }
+}
+
+/**
+ * The shader runtime is the base for any runtime associated with a @see ShaderBlock.
+ *
+ * It encapsulates the basic needs to render a full screen effect mainly the effect wrapper holding on the shader program.
+ *
+ * It is able to either render to a texture or directly to the main canvas.
+ *
+ * It also manages the disposal of the effect wrapper.
+ */
+export class ShaderRuntime implements IDisposable {
+    /**
+     * Promise that resolves when the effect is ready to be used.
+     */
+    public readonly onReadyAsync: Promise<void>;
+
+    private readonly _engine: AbstractEngine;
+    private readonly _effectRenderer: EffectRenderer;
+    private readonly _effectWrapper: EffectWrapper;
+    private readonly _shaderBinding: Binding;
+
+    /**
+     * Creates a new @see ShaderRuntime.
+     * @param effectRenderer - defines the effect renderer to use to render the full screen effect
+     * @param shaderProgram - defines the shader code associated with this runtime
+     * @param shaderBinding - defines the shader bindings associated with this runtime
+     */
+    constructor(effectRenderer: EffectRenderer, shaderProgram: ShaderProgram, shaderBinding: Binding) {
+        this._engine = effectRenderer.engine;
+        this._effectRenderer = effectRenderer;
+        this._shaderBinding = shaderBinding;
+        this._effectWrapper = new EffectWrapper({
+            engine: this._engine,
+            ...getShaderCreateOptions(shaderProgram),
+        });
+
+        // Wraps the effect readiness in a promise to expose it as a public property.
+        this.onReadyAsync = new Promise<void>((resolve, reject) => {
+            this._effectWrapper.effect.executeWhenCompiled(() => {
+                resolve();
+            });
+
+            this._effectWrapper.effect.onErrorObservable.addOnce((error) => {
+                reject(error);
+            });
+        });
+    }
+
+    /**
+     * Renders the full screen effect into a texture.
+     * @param renderTargetWrapper - The render target wrapper to render into
+     */
+    public renderToTexture(renderTargetWrapper: RenderTargetWrapper): void {
+        this._engine.bindFramebuffer(renderTargetWrapper);
+        this._draw(renderTargetWrapper.width, renderTargetWrapper.height);
+        this._engine.unBindFramebuffer(renderTargetWrapper);
+    }
+
+    /**
+     * Renders the full screen effect into the main canvas.
+     */
+    public renderToCanvas(): void {
+        this._effectRenderer.setViewport();
+        this._draw(this._engine.getRenderWidth(), this._engine.getRenderHeight());
+    }
+
+    /**
+     * "Draws" the full screen effect into the currently bound output.
+     * @param width - defines the width of the output
+     * @param height - defines the height of the output
+     */
+    private _draw(width: number, height: number): void {
+        this._effectRenderer.applyEffectWrapper(this._effectWrapper);
+        this._shaderBinding.bind(this._effectWrapper.effect, width, height);
+        this._effectRenderer.draw();
+    }
+
+    /**
+     * Disposes the runtime resources.
+     */
+    public dispose(): void {
+        this._effectWrapper.dispose();
+    }
+}

package/src/runtime/smartFilterRuntime.ts

@@ -0,0 +1,104 @@
+import { EffectRenderer } from "@babylonjs/core/Materials/effectRenderer.js";
+import type { ThinEngine } from "@babylonjs/core/Engines/thinEngine";
+import { CommandBuffer } from "../command/commandBuffer.js";
+import type { IDisposable } from "../IDisposable";
+import type { Command } from "../command/command";
+
+/**
+ * A runtime is a snapshot of a smart filter containing all the
+ * required data to render it as well as the entire command buffer.
+ */
+export type SmartFilterRuntime = {
+    /**
+     * The command buffer containing all the commands to execute during a frame.
+     */
+    readonly commandBuffer: Readonly<CommandBuffer>;
+
+    /**
+     * Renders one frame of the smart filter.
+     */
+    render(): void;
+
+    /**
+     * Dispose the runtime and all its associated resources
+     */
+    dispose(): void;
+};
+
+/**
+ * The internal runtime implementation exposing more information than @see SmartFilterRuntime.
+ * This is used internally to render the smart filter.
+ *
+ * It is not fully exposed publicly to prevent any misuse of the runtime.
+ */
+export class InternalSmartFilterRuntime implements SmartFilterRuntime {
+    /**
+     * The engine used by the smart filter.
+     */
+    public readonly engine: ThinEngine;
+
+    /**
+     * The effect renderer used by the smart filter.
+     */
+    public readonly effectRenderer: EffectRenderer;
+
+    /**
+     * The command buffer containing all the commands to execute during a frame.
+     */
+    public readonly commandBuffer: CommandBuffer;
+
+    private readonly _resources: IDisposable[];
+
+    /**
+     * Instantiates a new smart filter runtime for one given engine.
+     * @param engine - the engine to use to render the smart filter
+     */
+    constructor(engine: ThinEngine) {
+        this.engine = engine;
+
+        this._resources = [];
+
+        this.commandBuffer = new CommandBuffer();
+
+        this.effectRenderer = new EffectRenderer(engine);
+        this.registerResource(this.effectRenderer);
+    }
+
+    /**
+     * Reference a resource to be disposed when the runtime is disposed.
+     * @param resource - defines the resource to dispose once the runtime is disposed
+     */
+    public registerResource(resource: IDisposable): void {
+        this._resources.push(resource);
+    }
+
+    /**
+     * Registers a command to be executed during the render loop.
+     * @param command - defines the command to execute
+     */
+    public registerCommand(command: Command): void {
+        this.commandBuffer.push(command);
+    }
+
+    /**
+     * Renders the smart filter.
+     * This function will execute all the commands contained int the command buffer.
+     */
+    public render(): void {
+        try {
+            this.commandBuffer.execute();
+        } catch (e) {
+            // eslint-disable-next-line no-debugger
+            debugger;
+        }
+    }
+
+    /**
+     * Dispose the runtime and all its associated resources
+     */
+    public dispose(): void {
+        for (const resource of this._resources) {
+            resource.dispose();
+        }
+    }
+}

package/src/runtime/strongRef.ts

@@ -0,0 +1,18 @@
+/**
+ * This describes a strong reference to any data type.
+ */
+export type StrongRef<T> = {
+    /**
+     * The value of the strong reference.
+     */
+    value: T;
+};
+
+/**
+ * Create a strong reference to the given value.
+ * @param value - the value to wrap in a strong reference
+ * @returns the strong reference containing the value
+ */
+export function createStrongRef<T>(value: T): StrongRef<T> {
+    return { value };
+}