@babylonjs/smart-filters 0.1.0-alpha

package/src/smartFilter.ts

@@ -0,0 +1,227 @@
+import type { ThinEngine } from "@babylonjs/core/Engines/thinEngine";
+import type { Nullable } from "@babylonjs/core/types";
+import type { SmartFilterRuntime } from "./runtime/smartFilterRuntime";
+import type { BaseBlock } from "./blocks/baseBlock";
+import type { ConnectionPointType } from "./connection/connectionPointType";
+import type { ConnectionPoint } from "./connection/connectionPoint";
+import { OutputBlock } from "./blocks/outputBlock.js";
+import { InternalSmartFilterRuntime } from "./runtime/smartFilterRuntime.js";
+import { RenderTargetGenerator } from "./runtime/renderTargetGenerator.js";
+import { AggregateBlock } from "./blocks/aggregateBlock.js";
+
+/**
+ * How long to wait for shader compilation and texture loading to complete before erroring out.
+ */
+const InitializationTimeout = 10000;
+
+/**
+ * Data passed to the initialize function of the blocks.
+ */
+export type InitializationData = {
+    /**
+     * The current smart filter runtime the block is being initialized for.
+     */
+    readonly runtime: InternalSmartFilterRuntime;
+    /**
+     * The output block of the smart filter.
+     * This is used to determine if a block is linked to the output block so that we can prevent an
+     * extra render pass.
+     */
+    readonly outputBlock: OutputBlock;
+    /**
+     * The list of promises to wait for during the initialization step.
+     */
+    readonly initializationPromises: Promise<void>[];
+};
+
+/**
+ * The smart filter class is the main class of the smart filter module.
+ *
+ * It is responsible for managing a graph of smart filter blocks.
+ *
+ * It is also responsible for creating the runtime associated to the current state of the filter.
+ */
+export class SmartFilter {
+    /**
+     * The friendly name of the smart filter.
+     */
+    public readonly name: string;
+
+    /**
+     * The smart filter output (input connection point of the output block...).
+     *
+     * This is where the smart filter final block should be connected to in order to be visible on screen.
+     */
+    public readonly output: ConnectionPoint<ConnectionPointType.Texture>;
+
+    /**
+     * User defined comments to describe the current smart filter.
+     */
+    public comments: Nullable<string> = null;
+
+    /**
+     * Data used by the smart filter editor.
+     * TODO. strong type and hide this.
+     */
+    public editorData: any = null;
+
+    private readonly _attachedBlocks: Array<BaseBlock>;
+    private readonly _outputBlock: OutputBlock;
+
+    /**
+     * Creates a new instance of a @see SmartFilter.
+     * @param name - The friendly name of the smart filter
+     */
+    constructor(name: string) {
+        this.name = name;
+
+        this._attachedBlocks = new Array<BaseBlock>();
+        this._outputBlock = new OutputBlock(this);
+        this.output = this._outputBlock.input;
+    }
+
+    /**
+     * @returns the list of blocks attached to the smart filter.
+     */
+    public get attachedBlocks(): ReadonlyArray<BaseBlock> {
+        return this._attachedBlocks;
+    }
+
+    /**
+     * @returns The current class name of the smart filter.
+     */
+    public getClassName(): string {
+        return "SmartFilter";
+    }
+
+    /**
+     * Registers a block to be part of this smart filter.
+     * @param block - The block to register on the smart filter
+     * @throws if the block is already registered on another smart filter
+     * @remarks This function will not register the block if it is already registered on the smart filter.
+     */
+    public registerBlock(block: BaseBlock): void {
+        // It is impossible to attach a block from another filter
+        if (block.smartFilter !== this) {
+            throw new Error("Block is not part of this smart filter");
+        }
+
+        // No need to attach a block several times
+        if (this._attachedBlocks.indexOf(block) !== -1) {
+            return;
+        }
+
+        // Add the block to the list of attached blocks
+        this._attachedBlocks.push(block);
+    }
+
+    /**
+     * Removes the block from the smart filter.
+     * @param block - The block to remove from the smart filter
+     * @remarks This function will disconnect the block on removal.
+     * This Output block cannot be removed.
+     */
+    public removeBlock(block: BaseBlock): void {
+        const attachedBlockIndex = this._attachedBlocks.indexOf(block);
+
+        // The block can only be removed if it is not the output block
+        // and if it is attached to the smart filter
+        if (attachedBlockIndex > -1 && !block.isOutput) {
+            // Disconnects all the connections of the block
+            block.disconnect();
+
+            // Removes the block from the list of attached blocks
+            this._attachedBlocks.splice(attachedBlockIndex, 1);
+        }
+    }
+
+    private _generateCommandsAndGatherInitPromises(initializationData: InitializationData): void {
+        const outputBlock = this._outputBlock;
+
+        outputBlock.visit(initializationData, (block: BaseBlock, initializationData: InitializationData) => {
+            // If the block is the output block,
+            if (block === outputBlock) {
+                // We only need to do something if the connected block is an input block.
+                // Indeed, any other block linked to the output block would directly render to the canvas
+                // as an optimization.
+                // In case the output block is not linked to an input block, we do not need extra commands
+                // or resources to create a render pass.
+                if (outputBlock.input.connectedTo?.ownerBlock.isInput) {
+                    block.generateCommandsAndGatherInitPromises(initializationData, true);
+                }
+                return;
+            }
+
+            block.generateCommandsAndGatherInitPromises(
+                initializationData,
+                outputBlock.input.connectedTo?.ownerBlock === block
+            );
+        });
+    }
+
+    /**
+     * Create a new runtime for the current state of the smart filter.
+     * @param engine - The Babylon.js engine to use for the runtime
+     * @param renderTargetGenerator - The render target generator to use to generate the RTTs for the shader blocks. If not provided, a default one will be created.
+     * @returns the runtime that can be used to render the smart filter
+     */
+    public async createRuntimeAsync(
+        engine: ThinEngine,
+        renderTargetGenerator?: RenderTargetGenerator
+    ): Promise<SmartFilterRuntime> {
+        const runtime = new InternalSmartFilterRuntime(engine);
+
+        const initializationData: InitializationData = {
+            runtime,
+            outputBlock: this._outputBlock,
+            initializationPromises: [],
+        };
+
+        this._workWithAggregateFreeGraph(() => {
+            this._outputBlock.prepareForRuntime();
+
+            renderTargetGenerator = renderTargetGenerator ?? new RenderTargetGenerator(false);
+            renderTargetGenerator.setOutputTextures(this, initializationData);
+
+            this._outputBlock.propagateRuntimeData();
+
+            this._generateCommandsAndGatherInitPromises(initializationData);
+        });
+
+        // Wait for all the blocks to be initialized
+        if (initializationData.initializationPromises.length > 0) {
+            const timeoutPromise = new Promise((resolve) => setTimeout(resolve, InitializationTimeout, true));
+            const initializationPromises = Promise.all(initializationData.initializationPromises).then(() => false);
+            const timedOut = await Promise.race([initializationPromises, timeoutPromise]);
+            if (timedOut) {
+                throw new Error("Initialization promises timed out");
+            }
+        }
+
+        return runtime;
+    }
+
+    /**
+     * @internal
+     * Merges all aggregate blocks into the smart filter graph, executes the passed-in work, then restores the aggregate blocks.
+     * @param work - The work to execute with the aggregate blocks merged
+     */
+    public _workWithAggregateFreeGraph(work: () => void): void {
+        const mergedAggregateBlocks: AggregateBlock[] = [];
+
+        // Merge all aggregate blocks
+        this._outputBlock.visit({}, (block: BaseBlock, _extraData: Object) => {
+            if (block instanceof AggregateBlock) {
+                block._mergeIntoSmartFilter(mergedAggregateBlocks);
+            }
+        });
+
+        // Do the passed in work
+        work();
+
+        // Restore all aggregate blocks
+        for (const block of mergedAggregateBlocks) {
+            block._unmergeFromSmartFilter();
+        }
+    }
+}

package/src/utils/shaderCodeUtils.ts

@@ -0,0 +1,242 @@
+import type { Effect } from "@babylonjs/core/Materials/effect";
+
+/**
+ * The shader code decorator.
+ * Used to decorate the names of uniform, function and const variables for easier parsing.
+ */
+export const decorateChar = "_";
+
+/**
+ * Describes a shader function.
+ */
+export type ShaderFunction = {
+    /**
+     * The name of the function.
+     */
+    name: string;
+
+    /**
+     * The code of the function.
+     */
+    code: string;
+};
+
+/**
+ * Describes a shader code.
+ */
+export type ShaderCode = {
+    /**
+     * The declaration of the const variables.
+     */
+    const?: string;
+
+    /**
+     * The declaration of the uniform variables.
+     */
+    uniform?: string;
+
+    /**
+     * The declaration of the uniform variables that should be common for all ShaderBlock instances using this shader code.
+     */
+    uniformSingle?: string;
+
+    /**
+     * The name of the main function.
+     */
+    mainFunctionName: string;
+
+    /**
+     * The name of the main input texture.
+     */
+    mainInputTexture?: string;
+
+    /**
+     * The list of functions used in the shader.
+     */
+    functions: ShaderFunction[];
+};
+
+/**
+ * Describes a shader program.
+ */
+export type ShaderProgram = {
+    /**
+     * The vertex shader code.
+     */
+    vertex?: string | undefined;
+
+    /**
+     * The fragment shader code.
+     */
+    fragment: ShaderCode;
+};
+
+/**
+ * The list of options required to create a shader block.
+ * It mainly contains the shader code to execute and its associated parameters.
+ */
+export type ShaderCreationOptions = {
+    /**
+     * A friendly name for the shader visible in Spector or in logs.
+     */
+    name: string;
+    /**
+     * Fragment shader source code.
+     */
+    fragmentShader: string;
+    /**
+     * Vertex shader source code.
+     */
+    vertexShader?: string;
+    /**
+     * Attributes to use in the shader
+     */
+    attributeNames?: Array<string>;
+    /**
+     * Uniforms to use in the shader
+     */
+    uniformNames?: Array<string>;
+    /**
+     * Texture sampler names to use in the shader
+     */
+    samplerNames?: Array<string>;
+    /**
+     * Defines to use in the shader
+     */
+    defines?: Array<string>;
+    /**
+     * Callback when the effect is compiled
+     */
+    onCompiled?: (effect: Effect) => void;
+};
+
+/**
+ * Injects the disable uniform into a shader program.
+ * @param shaderProgram - The shader program to inject the disable uniform into.
+ * @returns The modified shader program
+ */
+export function injectDisableUniform(shaderProgram: ShaderProgram) {
+    const shaderFragment = shaderProgram.fragment;
+
+    shaderFragment.uniform += "\nuniform bool _disabled_;";
+
+    for (let i = 0; i < shaderFragment.functions.length; ++i) {
+        const func = shaderFragment.functions[i]!;
+        if (func.name === shaderFragment.mainFunctionName) {
+            func.code = func.code.replace(
+                "{",
+                `{\n    if (_disabled_) return texture2D(${shaderProgram.fragment.mainInputTexture}, vUV);\n`
+            );
+            break;
+        }
+    }
+
+    return shaderProgram;
+}
+
+/**
+ * Gets the shader fragment code.
+ * @param shaderProgram - The shader program to extract the code from.
+ * @param mainCodeOnly - If true, only the main function code will be returned.
+ * @returns The shader fragment code.
+ */
+export function getShaderFragmentCode(shaderProgram: ShaderProgram, mainCodeOnly = false): string {
+    const shaderFragment = shaderProgram.fragment;
+
+    const declarations =
+        (shaderFragment.const ?? "") +
+        "\n" +
+        shaderFragment.uniform +
+        "\n" +
+        (shaderFragment.uniformSingle ?? "") +
+        "\n";
+
+    let mainFunctionCode = "";
+    let otherFunctionsCode = "";
+    for (let i = 0; i < shaderFragment.functions.length; ++i) {
+        const func = shaderFragment.functions[i]!;
+        if (func.name === shaderFragment.mainFunctionName) {
+            mainFunctionCode += func.code + "\n";
+            if (mainCodeOnly) {
+                break;
+            }
+        } else {
+            otherFunctionsCode += func.code + "\n";
+        }
+    }
+
+    return mainCodeOnly ? mainFunctionCode : declarations + otherFunctionsCode + mainFunctionCode;
+}
+
+/**
+ * Gets the shader creation options from a shader program.
+ * @param shaderProgram - The shader program to build the create options from.
+ * @returns The shader creation options.
+ */
+export function getShaderCreateOptions(shaderProgram: ShaderProgram): ShaderCreationOptions {
+    const shaderFragment = shaderProgram.fragment;
+
+    let code = getShaderFragmentCode(shaderProgram);
+
+    const uniforms = shaderFragment.uniform + "\n" + (shaderFragment.uniformSingle ?? "");
+    const uniformNames = [];
+    const samplerNames = [];
+
+    const rx = new RegExp(`uniform\\s+(\\S+)\\s+(\\w+)\\s*;`, "g");
+
+    let match = rx.exec(uniforms);
+    while (match !== null) {
+        const varType = match[1]!;
+        const varName = match[2]!;
+
+        if (varType === "sampler2D" || varType === "sampler3D") {
+            samplerNames.push(varName);
+        } else {
+            uniformNames.push(varName);
+        }
+
+        match = rx.exec(uniforms);
+    }
+
+    code =
+        "varying vec2 vUV;\n" +
+        code +
+        "\nvoid main(void) {\ngl_FragColor = " +
+        shaderFragment.mainFunctionName +
+        "(vUV);\n}";
+
+    const options: ShaderCreationOptions = {
+        name: shaderFragment.mainFunctionName,
+        fragmentShader: code,
+        uniformNames: uniformNames,
+        samplerNames: samplerNames,
+    };
+
+    if (shaderProgram.vertex) {
+        options.vertexShader = shaderProgram.vertex;
+    }
+
+    return options;
+}
+
+/**
+ * Decorates a symbol (uniform, function or const) name.
+ * @param symbol - The symbol to decorate.
+ * @returns The decorated symbol.
+ */
+export function decorateSymbol(symbol: string): string {
+    return decorateChar + symbol + decorateChar;
+}
+
+/**
+ * Undecorates a symbol (uniform, function or const) name.
+ * @param symbol - The symbol to undecorate.
+ * @returns The undecorated symbol. Throws an error if the symbol is not decorated.
+ */
+export function undecorateSymbol(symbol: string): string {
+    if (symbol.charAt(0) !== decorateChar || symbol.charAt(symbol.length - 1) !== decorateChar) {
+        throw new Error(`undecorateSymbol: Invalid symbol name "${symbol}"`);
+    }
+
+    return symbol.substring(1, symbol.length - 1);
+}

package/src/utils/textureLoaders.ts

@@ -0,0 +1,28 @@
+import { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture.js";
+import { type ThinEngine } from "@babylonjs/core/Engines/thinEngine.js";
+
+/**
+ * Helper that takes in a URL to an image and returns a ThinTexture
+ * @param engine - defines the engine to use to create the texture
+ * @param url - defines a value which contains one of the following:
+ * * A conventional http URL, e.g. 'http://...' or 'file://...'
+ * * A base64 string of in-line texture data, e.g. 'data:image/jpg;base64,/...'
+ * @param flipY - Indicates if the Y axis should be flipped
+ * @param samplingMode - The sampling mode to use
+ * @returns A ThinTexture of the image
+ */
+export function createImageTexture(
+    engine: ThinEngine,
+    url: string,
+    flipY: boolean = true,
+    samplingMode: number | undefined = undefined
+): ThinTexture {
+    const internalTexture = engine.createTexture(url, true, flipY, null, samplingMode);
+    return new ThinTexture(internalTexture);
+}
+
+/*
+    Future util ideas:
+        HtmlElementTexture
+        WebCamTexture
+*/