@babylonjs/smart-filters 0.1.0-alpha

package/src/blocks/copyBlock.ts

@@ -0,0 +1,103 @@
+import type { Effect } from "@babylonjs/core/Materials/effect";
+import type { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture";
+
+import type { ShaderProgram } from "../utils/shaderCodeUtils";
+import type { SmartFilter } from "../smartFilter";
+import type { StrongRef } from "../runtime/strongRef";
+import { ConnectionPointType } from "../connection/connectionPointType.js";
+import { ShaderBlock } from "./shaderBlock.js";
+import { ShaderBinding } from "../runtime/shaderRuntime.js";
+
+const shaderProgram: ShaderProgram = {
+    fragment: {
+        uniform: `
+            uniform sampler2D _input_;
+        `,
+
+        mainFunctionName: "_copy_",
+
+        functions: [
+            {
+                name: "_copy_",
+                code: `
+                    vec4 _copy_(vec2 vUV) {
+                        return texture2D(_input_, vUV);
+                    }
+                `,
+            },
+        ],
+    },
+};
+
+/**
+ * The shader bindings for the Copy block.
+ */
+export class CopyShaderBinding extends ShaderBinding {
+    private readonly _inputTexture: StrongRef<ThinTexture>;
+
+    /**
+     * Creates a new shader binding instance for the copy block.
+     * @param parentBlock - The parent block
+     * @param inputTexture - defines the input texture to copy
+     */
+    constructor(parentBlock: CopyBlock, inputTexture: StrongRef<ThinTexture>) {
+        super(parentBlock);
+        this._inputTexture = inputTexture;
+    }
+
+    /**
+     * Binds all the required data to the shader when rendering.
+     * @param effect - defines the effect to bind the data to
+     * @internal
+     */
+    public override bind(effect: Effect): void {
+        effect.setTexture(this.getRemappedName("input"), this._inputTexture.value);
+    }
+}
+
+/**
+ * A block responsible for copying a texture to the output.
+ *
+ * This might be helpful to duplicate a texture if necessary.
+ *
+ * It simply takes a texture as input and outputs it to another texture or the main canvas.
+ */
+export class CopyBlock extends ShaderBlock {
+    /**
+     * The class name of the block.
+     */
+    public static override ClassName = "CopyBlock";
+
+    /**
+     * The input connection point of the block.
+     */
+    public readonly input = this._registerInput("input", ConnectionPointType.Texture);
+
+    /**
+     * The shader program (vertex and fragment code) to use to render the block
+     */
+    public static override ShaderCode = shaderProgram;
+
+    /**
+     * Create a new copy block.
+     * @param smartFilter - The smart filter this block belongs to.
+     * @param name -The friendly name of the block.
+     */
+    constructor(smartFilter: SmartFilter, name: string) {
+        super(smartFilter, name);
+    }
+
+    /**
+     * Get the class instance that binds all the required data to the shader (effect) when rendering.
+     * @returns The class instance that binds the data to the effect
+     */
+    public getShaderBinding(): ShaderBinding {
+        const input = this.input.runtimeData;
+
+        if (!input) {
+            throw new Error(`The input texture is missing for the CopyBlock named ${this.name}`);
+        }
+
+        return new CopyShaderBinding(this, input);
+    }
+}

package/src/blocks/disableableBlock.ts

@@ -0,0 +1,40 @@
+import type { SmartFilter } from "../smartFilter";
+import type { ConnectionPoint } from "../connection/connectionPoint";
+
+import { BaseBlock } from "../blocks/baseBlock.js";
+import { ConnectionPointType } from "../connection/connectionPointType.js";
+import { createStrongRef } from "../runtime/strongRef.js";
+
+/**
+ * The interface that describes the disableable block.
+ */
+export interface IDisableableBlock {
+    /**
+     * The disabled connection point of the block.
+     */
+    disabled: ConnectionPoint<ConnectionPointType.Boolean>;
+}
+
+/**
+ * A block that can be disabled.
+ */
+export class DisableableBlock extends BaseBlock implements IDisableableBlock {
+    /**
+     * The disabled connection point of the block.
+     */
+    public readonly disabled = this._registerOptionalInput(
+        "disabled",
+        ConnectionPointType.Boolean,
+        createStrongRef(false)
+    );
+
+    /**
+     * Instantiates a new block.
+     * @param smartFilter - Defines the smart filter the block belongs to
+     * @param name - Defines the name of the block
+     * @param disableOptimization - Defines if the block should not be optimized (default: false)
+     */
+    constructor(smartFilter: SmartFilter, name: string, disableOptimization = false) {
+        super(smartFilter, name, disableOptimization);
+    }
+}

package/src/blocks/inputBlock.ts

@@ -0,0 +1,114 @@
+import type { SmartFilter } from "../smartFilter";
+import type { ConnectionPointValue } from "../connection/connectionPointType";
+import type { RuntimeData } from "../connection/connectionPoint";
+import type { ConnectionPointWithDefault } from "../connection/connectionPointWithDefault";
+import type { DisableableBlock } from "./disableableBlock";
+import { BaseBlock } from "../blocks/baseBlock.js";
+import { createStrongRef } from "../runtime/strongRef.js";
+import { ConnectionPointType } from "../connection/connectionPointType.js";
+
+/**
+ * Type predicate to check if value is a strong ref or a direct value
+ * @param value - The value to check
+ * @returns true if the value is a strong ref, otherwise false
+ */
+function isRuntimeData<U extends ConnectionPointType>(
+    value: ConnectionPointValue<U> | RuntimeData<U>
+): value is RuntimeData<U> {
+    return (value as RuntimeData<ConnectionPointType>).value !== undefined;
+}
+
+/**
+ * Predicate to check if a block is a texture input block.
+ * @param block - The block to check
+ * @returns true if the block is a texture input block, otherwise false
+ */
+export function isTextureInputBlock(block: BaseBlock): block is InputBlock<ConnectionPointType.Texture> {
+    return (block as InputBlock<ConnectionPointType.Texture>).type === ConnectionPointType.Texture;
+}
+
+/**
+ * Predicate to check if a block is a disableable block.
+ * @param block - The block to check
+ * @returns true if the block is a disableable block, otherwise false
+ */
+export function isDisableableBlock(block: BaseBlock): block is DisableableBlock {
+    return (block as DisableableBlock).disabled !== undefined;
+}
+
+/**
+ * This represents any inputs used in the graph.
+ *
+ * This is used to provide a way to connect the graph to the outside world.
+ *
+ * The value is dynamically set by the user.
+ */
+export class InputBlock<U extends ConnectionPointType> extends BaseBlock {
+    /**
+     * The class name of the block.
+     */
+    public static override ClassName = "InputBlock";
+
+    /**
+     * The output connection point of the block.
+     */
+    public readonly output: ConnectionPointWithDefault<U>;
+
+    /**
+     * The type of the input.
+     */
+    public readonly type: U;
+
+    /**
+     * Gets the current value of the input.
+     */
+    public get runtimeValue(): RuntimeData<U> {
+        return this.output.runtimeData;
+    }
+
+    /**
+     * Sets the current value of the input.
+     */
+    public set runtimeValue(value: RuntimeData<U>) {
+        this.output.runtimeData = value;
+    }
+
+    /**
+     * Creates a new InputBlock.
+     * @param smartFilter - The smart filter to add the block to
+     * @param name - The friendly name of the block
+     * @param type - The type of the input
+     * @param initialValue - The initial value of the input
+     * @remarks the initial value can either be a strong reference or a value
+     */
+    constructor(
+        smartFilter: SmartFilter,
+        name: string,
+        type: U,
+        initialValue: ConnectionPointValue<U> | RuntimeData<U>
+    ) {
+        super(smartFilter, name);
+        this.type = type;
+
+        // Creates the output connection point
+        this.output = this._registerOutputWithDefault(
+            "output",
+            type,
+            isRuntimeData(initialValue) ? initialValue : createStrongRef(initialValue)
+        );
+
+        // Creates a strong reference to the initial value in case a reference has not been provided
+        if (isRuntimeData(initialValue)) {
+            this.runtimeValue = initialValue;
+        } else {
+            this.runtimeValue = createStrongRef(initialValue);
+        }
+    }
+}
+
+/**
+ * Unionized type of all the possible input types.
+ */
+export type AnyInputBlock = {
+    [T in keyof typeof ConnectionPointType]: InputBlock<(typeof ConnectionPointType)[T]>;
+}[keyof typeof ConnectionPointType];

package/src/blocks/outputBlock.ts

@@ -0,0 +1,97 @@
+import type { InitializationData, SmartFilter } from "../smartFilter";
+import { ConnectionPointType } from "../connection/connectionPointType.js";
+import { BaseBlock } from "./baseBlock.js";
+import { CopyBlock } from "./copyBlock.js";
+import { ShaderRuntime } from "../runtime/shaderRuntime.js";
+import { createCommand } from "../command/command.js";
+
+/**
+ * The output block of a smart filter.
+ *
+ * Only the smart filter will internally create and host the output block.
+ * It should not be exported through the main index.ts module.
+ */
+export class OutputBlock extends BaseBlock {
+    /**
+     * The class name of the block.
+     */
+    public static override ClassName = "OutputBlock";
+
+    /**
+     * Input connection point of the output block.
+     * This takes a texture as input.
+     */
+    public readonly input = this._registerInput("input", ConnectionPointType.Texture);
+
+    private _copyBlock: CopyBlock | null;
+
+    /**
+     * Create a new output block.
+     * @param smartFilter - The smart filter this block belongs to
+     */
+    constructor(smartFilter: SmartFilter) {
+        super(smartFilter, "output");
+
+        this._copyBlock = null;
+    }
+
+    private _getCopyBlock(): CopyBlock {
+        if (!this._copyBlock) {
+            this._copyBlock = new CopyBlock(this.smartFilter, "copy");
+            this._copyBlock.input.runtimeData = this.input.runtimeData;
+        }
+
+        return this._copyBlock;
+    }
+
+    /**
+     * Prepares all blocks for runtime by traversing the graph.
+     */
+    public override prepareForRuntime(): void {
+        this.visit({}, (block: BaseBlock, _extraData: Object) => {
+            if (block !== this) {
+                block.prepareForRuntime();
+            }
+        });
+    }
+
+    /**
+     * Propagates the runtime data for all graph blocks.
+     */
+    public override propagateRuntimeData(): void {
+        this.visit({}, (block: BaseBlock, _extraData: Object) => {
+            if (block !== this) {
+                block.propagateRuntimeData();
+            }
+        });
+    }
+
+    /**
+     * Generates the commands needed to execute the block at runtime and gathers promises for initialization work
+     * @param initializationData - The initialization data to use
+     * @param finalOutput - Defines if the block is the final output of the smart filter
+     */
+    public override generateCommandsAndGatherInitPromises(
+        initializationData: InitializationData,
+        finalOutput: boolean
+    ): void {
+        const copyBlock = this._getCopyBlock();
+        const runtime = initializationData.runtime;
+
+        const shaderBlockRuntime = new ShaderRuntime(
+            runtime.effectRenderer,
+            copyBlock.getShaderProgram(),
+            copyBlock.getShaderBinding()
+        );
+        initializationData.initializationPromises.push(shaderBlockRuntime.onReadyAsync);
+        runtime.registerResource(shaderBlockRuntime);
+
+        runtime.registerCommand(
+            createCommand(`${this.getClassName()}.renderToCanvas`, this, () => {
+                shaderBlockRuntime.renderToCanvas();
+            })
+        );
+
+        super.generateCommandsAndGatherInitPromises(initializationData, finalOutput);
+    }
+}

package/src/blocks/shaderBlock.ts

@@ -0,0 +1,145 @@
+import type { ThinRenderTargetTexture } from "@babylonjs/core/Materials/Textures/thinRenderTargetTexture";
+import "@babylonjs/core/Engines/Extensions/engine.renderTarget.js";
+
+import type { InitializationData, SmartFilter } from "../smartFilter";
+import type { ShaderProgram } from "../utils/shaderCodeUtils";
+import type { Binding } from "../runtime/shaderRuntime";
+import type { ConnectionPoint } from "../connection/connectionPoint";
+import { ShaderRuntime } from "../runtime/shaderRuntime.js";
+import { ConnectionPointType } from "../connection/connectionPointType.js";
+import { createCommand } from "../command/command.js";
+import { DisableableBlock } from "./disableableBlock.js";
+import { undecorateSymbol } from "../utils/shaderCodeUtils.js";
+
+/**
+ * This is the base class for all shader blocks.
+ *
+ * It contains the redundant part of wrapping a shader for a full screen pass.
+ *
+ * The only required function to implement is the bind function.
+ */
+export abstract class ShaderBlock extends DisableableBlock {
+    /**
+     * The class name of the block.
+     */
+    public static override ClassName = "ShaderBlock";
+
+    /**
+     * Get the class instance that binds all the required data to the shader (effect) when rendering.
+     * It should throw an error if required inputs are missing.
+     * @returns The class instance that binds the data to the effect
+     */
+    public abstract getShaderBinding(): Binding;
+
+    /**
+     * The shader program (vertex and fragment code) to use to render the block
+     */
+    public static ShaderCode: ShaderProgram;
+
+    /**
+     * The output connection point of the block.
+     */
+    public readonly output = this._registerOutput("output", ConnectionPointType.Texture);
+
+    protected _textureRatio: number = 1;
+
+    /**
+     * Gets the texture ratio of the output texture.
+     */
+    public get textureRatio() {
+        return this._textureRatio;
+    }
+
+    /**
+     * Sets the texture ratio of the output texture.
+     */
+    public set textureRatio(value: number) {
+        this._textureRatio = value;
+    }
+
+    /**
+     * Disconnects the block from the graph.
+     * @param disconnectedConnections - Stores the connections that have been broken in the process. You can reconnect them later if needed.
+     */
+    public override disconnectFromGraph(disconnectedConnections?: [ConnectionPoint, ConnectionPoint][]): void {
+        const input = this._getConnectionForMainInputTexture();
+
+        for (const endpoint of this.output.endpoints) {
+            disconnectedConnections?.push([endpoint, this.output]);
+            input.connectTo(endpoint);
+        }
+    }
+
+    protected _getConnectionForMainInputTexture(): ConnectionPoint {
+        const mainInputTextureName = this.getShaderProgram().fragment.mainInputTexture;
+        if (!mainInputTextureName) {
+            throw `The block named "${this.name}" does not have a main input texture defined!`;
+        }
+
+        const mainInputTexture = this.findInput(undecorateSymbol(mainInputTextureName));
+        if (!mainInputTexture || !mainInputTexture.connectedTo) {
+            throw `The main input texture "${mainInputTextureName}" of block named "${this.name}" is not connected!`;
+        }
+
+        return mainInputTexture.connectedTo;
+    }
+
+    /**
+     * Instantiates a new block.
+     * @param smartFilter - Defines the smart filter the block belongs to
+     * @param name - Defines the name of the block
+     * @param disableOptimization - Defines if the block should not be optimized (default: false)
+     */
+    constructor(smartFilter: SmartFilter, name: string, disableOptimization = false) {
+        super(smartFilter, name, disableOptimization);
+    }
+
+    /**
+     * Gets the shader program to use to render the block.
+     * @returns The shader program to use to render the block
+     */
+    public getShaderProgram() {
+        return (this.constructor as typeof ShaderBlock).ShaderCode;
+    }
+
+    /**
+     * Generates the commands needed to execute the block at runtime and gathers promises for initialization work
+     * @param initializationData - The initialization data to use
+     * @param finalOutput - Defines if the block is the final output of the smart filter
+     */
+    public override generateCommandsAndGatherInitPromises(
+        initializationData: InitializationData,
+        finalOutput: boolean
+    ): void {
+        const runtime = initializationData.runtime;
+        const shaderBlockRuntime = new ShaderRuntime(
+            runtime.effectRenderer,
+            this.getShaderProgram(),
+            this.getShaderBinding()
+        );
+        initializationData.initializationPromises.push(shaderBlockRuntime.onReadyAsync);
+        runtime.registerResource(shaderBlockRuntime);
+
+        if (finalOutput) {
+            runtime.registerCommand(
+                createCommand(`${this.getClassName()}.renderToCanvas`, this, () => {
+                    shaderBlockRuntime.renderToCanvas();
+                })
+            );
+        } else {
+            const rtt =
+                this.output.runtimeData && (this.output.runtimeData.value as ThinRenderTargetTexture).renderTarget;
+            if (!rtt) {
+                throw new Error("ShaderBlock does not have a render target texture.");
+            }
+
+            runtime.registerCommand(
+                createCommand(`${this.getClassName()}.render`, this, () => {
+                    shaderBlockRuntime.renderToTexture(rtt);
+                })
+            );
+        }
+
+        super.generateCommandsAndGatherInitPromises(initializationData, finalOutput);
+    }
+}

package/src/command/command.ts

@@ -0,0 +1,60 @@
+/**
+ * Represents the owner of a command.
+ * In practice, it will mostly be a block, the smart filter or a tool injecting commands.
+ */
+export interface ICommandOwner {
+    /**
+     * The friendly name of the owner.
+     */
+    readonly name: string;
+
+    /**
+     * Gets the class name of the owner.
+     */
+    getClassName(): string;
+}
+
+/**
+ * Represents a the action of a @see Command. This is what will be executed during a command buffer execution.
+ */
+export type CommandAction = () => void;
+
+/**
+ * Represents a command to execute.
+ *
+ * A command contains a function that will be executed at runtime by the smart filter.
+ *
+ * It also contains the owner of the command for debugging purposes.
+ */
+export type Command = {
+    /**
+     * The friendly name of the command.
+     */
+    readonly name: string;
+
+    /**
+     * The owner of the command.
+     * In practice, it will mostly be a block, the smart filter or a tool injecting commands.
+     */
+    readonly owner: ICommandOwner;
+
+    /**
+     * Defines the action to execute.
+     */
+    readonly action: CommandAction;
+};
+
+/**
+ * Creates a new command.
+ * @param name - The friendly name of the command
+ * @param owner - The owner of the command
+ * @param action - The action to execute when the command is executed
+ * @returns The new command
+ */
+export function createCommand(name: string, owner: ICommandOwner, action: CommandAction): Command {
+    return {
+        name,
+        owner,
+        action,
+    };
+}

package/src/command/commandBuffer.ts

@@ -0,0 +1,71 @@
+import type { Command } from "./command";
+
+/**
+ * Represents the action to run when calling `visitCommands` on a @see CommandBuffer.
+ */
+export type CommandsVisitor = (command: Command) => void;
+
+/**
+ * @see CommandsVisitor used to execute the commands of a @see CommandBuffer.
+ * @param command - The command to execute.
+ */
+function executeCommandsVisitor(command: Command) {
+    command.action();
+}
+
+/**
+ * A command buffer is a list of commands to execute.
+ * This is used to store the list of tasks the current smart filter needs to execute.
+ */
+export class CommandBuffer {
+    private readonly _commands: Command[] = [];
+
+    /**
+     * Creates a new command buffer.
+     * @param args - the list of commands to add to the command buffer
+     */
+    constructor(...args: Command[]) {
+        for (const command of args) {
+            this.push(command);
+        }
+    }
+
+    /**
+     * Adds a command to the command buffer.
+     * @param command - the command to add
+     */
+    public push(command: Command) {
+        this._commands.push(command);
+    }
+
+    /**
+     * Clears the command buffer and empty the list of commands.
+     */
+    public clear() {
+        this._commands.length = 0;
+    }
+
+    /**
+     * Visits all the commands in the command buffer.
+     * @param commandVisitor - The action to execute on each command
+     */
+    public visitCommands(commandVisitor: CommandsVisitor): void {
+        for (const command of this._commands) {
+            commandVisitor(command);
+        }
+    }
+
+    /**
+     * Execute all the commands in the command buffer.
+     */
+    public execute() {
+        this.visitCommands(executeCommandsVisitor);
+    }
+
+    /**
+     * Dispose the resources associated to the command buffer.
+     */
+    public dispose() {
+        this.clear();
+    }
+}

package/src/command/commandBufferDebugger.ts

@@ -0,0 +1,13 @@
+import type { CommandBuffer } from "./commandBuffer";
+
+/**
+ * Logs all the commands associated to a command buffer.
+ * @param commandBuffer - The command buffer to log
+ */
+export function logCommands(commandBuffer: Readonly<CommandBuffer>) {
+    console.log("----- Command buffer commands -----");
+    commandBuffer.visitCommands((command) => {
+        console.log(`  Owner: ${command.owner.getClassName()} (${command.owner.name}) - Command: ${command.name}`);
+    });
+    console.log("-----------------------------------");
+}