@needle-tools/materialx 1.0.1 → 1.0.2-next.c468cd8

package/src/materialx.ts

@@ -1,14 +1,15 @@
-import { Context, delay, isDevEnvironment, ObjectUtils } from "@needle-tools/engine";
+import { Context, delay, isDevEnvironment, ObjectUtils, GameObject, onBeforeRender } from "@needle-tools/engine";
+import type { MaterialX as MX } from "./materialx.types.js";
 import MaterialX from "../bin/JsMaterialXGenShader.js";
 import { debug } from "./utils.js";
 import { renderPMREMToEquirect } from "./textureHelper.js";
-import { Light, MeshBasicMaterial, Object3D, PMREMGenerator } from "three";
-import { registerLights } from "./helper.js";
+import { Light, Material, MeshBasicMaterial, Object3D, PMREMGenerator, Texture } from "three";
+import { registerLights, getLightData } from "./materialx.helper.js";
+import type { MaterialXMaterial } from "./materialx.material.js";
 
 
-// Global MaterialX module instance - initialized lazily
 export const state = new class {
-    materialXModule: any = null;
+    materialXModule: MX.MODULE | null = null;
     materialXGenerator: any = null;
     materialXGenContext: any = null;
     materialXStdLib: any = null;
@@ -22,14 +23,14 @@ export const state = new class {
     }
 }
 
-// Initialize MaterialX WASM module lazily
-export async function initializeMaterialX(): Promise<void> {
+/** Initialize the MaterialX module. Must be awaited before trying to create materials */
+export async function ready(): Promise<void> {
     if (state.materialXInitPromise) {
         return state.materialXInitPromise;
     }
     return state.materialXInitPromise = (async () => {
         if (state.materialXModule) return; // Already initialized
-        if (debug) console.log("Initializing MaterialX WASM module...");
+        if (debug) console.log("[MaterialX] Initializing WASM module...");
         try {
 
             const urls: Array<string> = await Promise.all([
@@ -44,7 +45,7 @@ export async function initializeMaterialX(): Promise<void> {
 
             const module = await MaterialX({
                 locateFile: (path: string, scriptDirectory: string) => {
-                    if (debug) console.debug("MaterialX locateFile called:", { path, scriptDirectory });
+                    if (debug) console.debug("[MaterialX]  locateFile called:", { path, scriptDirectory });
 
                     if (path.includes("JsMaterialXCore.wasm")) {
                         return JsMaterialXCore; // Use the URL for the core WASM file
@@ -59,8 +60,8 @@ export async function initializeMaterialX(): Promise<void> {
                     return scriptDirectory + path;
                 },
             });
-            if (debug) console.log("MaterialXLoader module loaded", module);
-            state.materialXModule = module;
+            if (debug) console.log("[MaterialX] module loaded", module);
+            state.materialXModule = module as MX.MODULE
 
             // Initialize shader generator and context
             state.materialXGenerator = module.EsslShaderGenerator.create();
@@ -71,158 +72,190 @@ export async function initializeMaterialX(): Promise<void> {
             state.materialXStdLib = module.loadStandardLibraries(state.materialXGenContext);
             tempDoc.setDataLibrary(state.materialXStdLib);
 
-            // Initialize basic lighting with default light rig
-            //             const defaultLightRigXml = `<?xml version="1.0"?>
-            // <materialx version="1.39">
-            //   <!-- Default directional light -->
-            //   <directional_light name="default_light" type="lightshader">
-            //     <input name="direction" type="vector3" value="0.0, -1.0, -0.5" />
-            //     <input name="color" type="color3" value="1.0, 1.0, 1.0" />
-            //     <input name="intensity" type="float" value="1.0" />
-            //   </directional_light>
-            // </materialx>`;
+            // TODO ShaderInterfaceType.SHADER_INTERFACE_REDUCED would be better, but doesn't actually seem to be supported in the MaterialX javascript bindings
+            state.materialXGenContext.getOptions().shaderInterfaceType = state.materialXModule.ShaderInterfaceType.SHADER_INTERFACE_COMPLETE;
+
+            // SPECULAR_ENVIRONMENT_NONE: Do not use specular environment maps.
+            // SPECULAR_ENVIRONMENT_FIS: Use Filtered Importance Sampling for specular environment/indirect lighting.
+            // SPECULAR_ENVIRONMENT_PREFILTER: Use pre-filtered environment maps for specular environment/indirect lighting.
+            state.materialXGenContext.getOptions().hwSpecularEnvironmentMethod = state.materialXModule.HwSpecularEnvironmentMethod.SPECULAR_ENVIRONMENT_FIS;
+
+            // TRANSMISSION_REFRACTION: Use a refraction approximation for transmission rendering.
+            // TRANSMISSION_OPACITY: Use opacity for transmission rendering.
+            // state.materialXGenContext.getOptions().hwTransmissionRenderMethod = state.materialXModule.HwTransmissionRenderMethod.TRANSMISSION_REFRACTION;
+
+            // Turned off because we're doing color space conversion the three.js way
+            state.materialXGenContext.getOptions().hwSrgbEncodeOutput = false;
+
+            // Enables the generation of a prefiltered environment map.
+            // TODO Would be great to use but requires setting more uniforms (like u_envPrefilterMip).
+            // When set to true, the u_envRadiance map is expected to be a prefiltered environment map.
+            // state.materialXGenContext.getOptions().hwWriteEnvPrefilter = true;
+
+            // Set a reasonable default for max active lights
+            state.materialXGenContext.getOptions().hwMaxActiveLightSources = 4;
 
             // This prewarms the shader generation context to have all light types
-            await registerLights(state.materialXModule, [], state.materialXGenContext);
+            await registerLights(state.materialXModule, state.materialXGenContext);
 
-            if (debug) console.log("MaterialX generator initialized successfully");
+            if (debug) console.log("[MaterialX] Generator initialized successfully");
         } catch (error) {
-            console.error("Failed to load MaterialX module:", error);
+            console.error("[MaterialX] Failed to load MaterialX module:", error);
             throw error;
         }
     })();
 }
 
+type EnvironmentTextureSet = {
+    radianceTexture: Texture | null;
+    irradianceTexture: Texture | null;
+}
+
 // MaterialX Environment Manager - handles lighting and environment setup
 export class MaterialXEnvironment {
-    private lights: any[] = [];
-    private lightData: any = null;
-    private radianceTexture: any = null;
-    private irradianceTexture: any = null;
-    private context: Context | null = null;
-    private initialized: boolean = false;
+    private _context: Context | null = null;
+    private _lights: Array<Light> = [];
+    private _lightData: any = null;
+    private _lightCount: number = 0;
+    private _initializePromise: Promise<boolean> | null = null;
+
+    private _unsubscribehook: (() => void) | null = null;
 
     constructor() {
-        if (debug) console.log("MaterialX Environment created");
+        if (debug) console.log("[MaterialX] Environment created");
     }
 
-    setContext(context: Context) {
-        this.context = context;
+    // Initialize with Needle Engine context
+    async initialize(context: Context): Promise<boolean> {
+        if (this._initializePromise) {
+            return this._initializePromise;
+        }
+        return this._initializePromise = this._initialize(context);
     }
 
-    /*
-    // Initialize MaterialX lighting system based on the reference implementation
-    async initializeLighting(lightRigXml: string, renderer?: any, radianceTexture?: any, irradianceTexture?: any): Promise<void> {
-        if (!materialXModule || !materialXGenContext) {
-            console.warn("MaterialX module not initialized, skipping lighting setup");
-            return;
+    get lightData() { return this._lightData; }
+    get lightCount() { return this._lightCount || 0; }
+    getTextures(material: MaterialXMaterial) {
+        if (material.envMap) {
+            // If the material has its own envMap, we don't use the irradiance texture
+            return this._getTextures(material.envMap);
         }
-
-        registerLights(materialXModule, this.lights, materialXGenContext);
+        return this._getTextures(this._context?.scene.environment);
     }
-    */
 
-    // Initialize with Needle Engine context
-    async initializeFromContext(): Promise<void> {
-        if (!this.context) {
-            console.warn("No Needle context available for MaterialX environment initialization");
-            return;
-        }
+    private _pmremGenerator: PMREMGenerator | null = null;
+    private readonly _texturesCache: Map<Texture | null, EnvironmentTextureSet> = new Map();
 
-        // Prevent multiple initializations
-        if (this.initialized) {
-            if (debug) console.log("MaterialX environment already initialized, skipping");
-            return;
-        }
+    private _initialize: (context: Context) => Promise<boolean> = async (context: Context) => {
 
-        // Clean up previous textures if they exist
-        if (this.radianceTexture) {
-            if (debug) console.log("Disposing previous radiance texture");
-            this.radianceTexture.dispose();
-            this.radianceTexture = null;
-        }
-        if (this.irradianceTexture) {
-            if (debug) console.log("Disposing previous irradiance texture");
-            this.irradianceTexture.dispose();
-            this.irradianceTexture = null;
-        }
+        this._context = context;
+        this._pmremGenerator = new PMREMGenerator(context.renderer);
 
-        // Get renderer from context
-        const renderer = this.context.renderer;
+        this._unsubscribehook?.();
+        this._unsubscribehook = onBeforeRender(() => {
+            this.updateLighting(false);
+            this._getTextures(context.scene.environment);
+        })
 
         // TODO remove this delay; we should wait for the scene lighting to be ready
         // and then update the uniforms
-        let envMap = this.context.scene.environment;
-        while (!envMap) {
-            await delay(200);
-            envMap = this.context.scene.environment;
-        }
-        var pmrem = new PMREMGenerator(renderer);
-        const target = pmrem.fromEquirectangular(envMap);
-
-        const radianceRenderTarget = renderPMREMToEquirect(renderer, target.texture, 0.0, 1024, 512, target.height);
-        const irradianceRenderTarget = renderPMREMToEquirect(renderer, target.texture, 1.0, 32, 16, target.height);
-
-        this.radianceTexture = radianceRenderTarget.texture;
-        this.irradianceTexture = irradianceRenderTarget.texture;
-
-        // Clean up PMREM generator and its render target
-        target.dispose();
-        pmrem.dispose();
-
-        if (debug) {
-            console.log({ radiance: this.radianceTexture, irradiance: this.irradianceTexture });
-            // Show both of them on cubes in the scene
-            const unlitMat = new MeshBasicMaterial();
-            const radianceMat = unlitMat.clone();
-            radianceMat.map = this.radianceTexture;
-            const radianceCube = ObjectUtils.createPrimitive("Cube", { material: radianceMat });
-            const irradianceMat = unlitMat.clone();
-            irradianceMat.map = this.irradianceTexture;
-            const irradianceCube = ObjectUtils.createPrimitive("Cube", { material: irradianceMat });
-            this.context.scene.add(radianceCube);
-            this.context.scene.add(irradianceCube);
-            radianceCube.position.set(2, 0, 0);
-            radianceCube.scale.y = 0.00001;
-            irradianceCube.position.set(-2, 0, 0);
-            irradianceCube.scale.y = 0.00001;
-            // await this.initializeLighting(defaultLightRigXml, renderer);
-            console.log("MaterialX environment initialized from Needle context", this, this.context.scene);
+        while (!context.scene.environment) {
+            await delay(5);
         }
+        this._getTextures(context.scene.environment);
+
+        // if (debug) {
+        //     console.log({ radiance: this._radianceTexture, irradiance: this._irradianceTexture });
+        //     // Show both of them on cubes in the scene
+        //     const unlitMat = new MeshBasicMaterial();
+        //     unlitMat.side = 2;
+        //     const radianceMat = unlitMat.clone();
+        //     radianceMat.map = this._radianceTexture;
+        //     const radianceCube = ObjectUtils.createPrimitive("Quad", { material: radianceMat });
+        //     const irradianceMat = unlitMat.clone();
+        //     irradianceMat.map = this._irradianceTexture;
+        //     const irradianceCube = ObjectUtils.createPrimitive("Quad", { material: irradianceMat });
+        //     context.scene.add(radianceCube);
+        //     context.scene.add(irradianceCube);
+        //     radianceCube.position.set(2, 0, 0);
+        //     irradianceCube.position.set(-2, 0, 0);
+        //     console.log("[MaterialX] environment initialized from Needle context", this, this._context.scene);
+        // }
+
+        this.updateLighting(true);
 
-        // Find lights in scene
-        let lights = new Array<Light>();
-        this.context.scene.traverse((object: Object3D) => {
-            if ((object as Light).isLight) lights.push(object as Light);
-        });
+        // Mark as initialized
+        return true;
+    }
 
-        this.lightData = await registerLights(state.materialXModule, lights, state.materialXGenContext);
+    // Reset the environment to allow re-initialization
+    reset() {
+        if (debug) console.log("[MaterialX] Resetting environment");
+        this._initializePromise = null;
+        this._lights = [];
+        this._lightData = null;
+        this._lightCount = 0;
+        this._pmremGenerator?.dispose();
+        this._pmremGenerator = null;
+        for(const textureSet of this._texturesCache.values()) {
+            textureSet.radianceTexture?.dispose();
+            textureSet.irradianceTexture?.dispose();
+        }
+        this._texturesCache.clear();
 
-        // Mark as initialized
-        this.initialized = true;
+        this._unsubscribehook?.();
+        this._unsubscribehook = null;
     }
 
-    // getLights() { return this.lights; }
-    getLightData() { return this.lightData; }
-    getRadianceTexture() { return this.radianceTexture; }
-    getIrradianceTexture() { return this.irradianceTexture; }
+    private _getTextures(texture: Texture | null | undefined): {
+        radianceTexture: Texture | null,
+        irradianceTexture: Texture | null
+    } {
 
-    setRadianceTexture(texture: any) { this.radianceTexture = texture; }
-    setIrradianceTexture(texture: any) { this.irradianceTexture = texture; }
+        let res: EnvironmentTextureSet | undefined = this._texturesCache.get(texture || null);
+        if (res) {
+            return res;
+        }
 
-    // Reset the environment to allow re-initialization
-    reset() {
-        if (debug) console.log("Resetting MaterialX environment");
-        if (this.radianceTexture) {
-            this.radianceTexture.dispose();
-            this.radianceTexture = null;
+        if (this._context && this._pmremGenerator && texture) {
+            if (debug) console.log("[MaterialX] Generating environment textures", texture.name);
+
+            const target = this._pmremGenerator.fromEquirectangular(texture);
+            const radianceRenderTarget = renderPMREMToEquirect(this._context.renderer, target.texture, 0.0, 1024, 512, target.height);
+            const irradianceRenderTarget = renderPMREMToEquirect(this._context.renderer, target.texture, 1.0, 32, 16, target.height);
+            target.dispose();
+            res = {
+                radianceTexture: radianceRenderTarget.texture,
+                irradianceTexture: irradianceRenderTarget.texture
+            }
         }
-        if (this.irradianceTexture) {
-            this.irradianceTexture.dispose();
-            this.irradianceTexture = null;
+        else {
+            res = {
+                radianceTexture: null,
+                irradianceTexture: null
+            }
+        }
+        this._texturesCache.set(texture || null, res);
+        return res;
+    }
+
+    private updateLighting = (collectLights: boolean = false) => {
+        if (!this._context) return;
+
+        // Find lights in scene
+        if (collectLights) {
+            const lights = new Array<Light>();
+            this._context.scene.traverse((object: Object3D) => {
+                if ((object as Light).isLight && GameObject.isActiveInHierarchy(object))
+                    lights.push(object as Light);
+            });
+            this._lights = lights;
+        }
+
+        if (state.materialXGenContext) {
+            const { lightData, lightCount } = getLightData(this._lights, state.materialXGenContext);
+            this._lightData = lightData;
+            this._lightCount = lightCount;
         }
-        this.initialized = false;
-        // this.lights = [];
-        this.lightData = null;
     }
 }

package/src/materialx.types.d.ts

@@ -0,0 +1,50 @@
+
+
+
+export namespace MaterialX {
+
+    export type MODULE = {
+        ShaderInterfaceType: any;
+        HwSpecularEnvironmentMethod: any;
+        HwShaderGenerator: {
+            bindLightShader(def: any, id: number, genContext: GenContext): void;
+            unbindLightShaders(context: any): void;
+        };
+        createDocument(): Document;
+        readFromXmlString(doc: Document, xml: string, unknown: string): void;
+        loadStandardLibraries(genContext: GenContext): StandardLibrary;
+        isTransparentSurface(renderableElement: any, target: string): boolean;
+    }
+
+
+    export type GenContext = {
+    }
+
+    export type StandardLibrary = {
+
+    }
+    
+    // https://github.com/AcademySoftwareFoundation/MaterialX/blob/b74787db6544283dc32afc8085ebc93cabe937cb/source/MaterialXGenShader/ShaderStage.h#L56
+    export type ShaderStage = {
+        getUniformBlocks(): Record<string, any>;
+    }
+
+    export type Document = {
+        setDataLibrary(lib: StandardLibrary): void;
+        importLibrary(lib: Document): void;
+
+        getNodes(): Node[];
+    }
+
+    export type Node = {
+        getType(): string;
+    }
+
+    export type Matrix = {
+        numRows(): number;
+        numColumns(): number;
+        get size(): number;
+        getItem(row: number, col: number): number;
+    }
+
+}

package/src/textureHelper.ts

@@ -40,7 +40,7 @@ export function renderPMREMToEquirect(renderer: WebGLRenderer, pmremTexture: Tex
     } else {
         imageHeight = 256; // Final fallback
     }
-    
+
     const maxMip = Math.log2(imageHeight) - 2;
     const cubeUVHeight = imageHeight;
     const cubeUVWidth = 3 * Math.max(Math.pow(2, maxMip), 7 * 16);
@@ -129,14 +129,14 @@ export function renderPMREMToEquirect(renderer: WebGLRenderer, pmremTexture: Tex
     const currentAutoClear = renderer.autoClear;
     const currentXrEnabled = renderer.xr.enabled;
     const currentShadowMapEnabled = renderer.shadowMap.enabled;
-    
+
     renderTarget.texture.generateMipmaps = true;
-    
+
     try {
         // Disable XR and shadow mapping during our render to avoid interference
         renderer.xr.enabled = false;
         renderer.shadowMap.enabled = false;
-        
+
         // Render to our target
         renderer.autoClear = true;
         renderer.setRenderTarget(renderTarget);
@@ -148,7 +148,7 @@ export function renderPMREMToEquirect(renderer: WebGLRenderer, pmremTexture: Tex
         renderer.autoClear = currentAutoClear;
         renderer.xr.enabled = currentXrEnabled;
         renderer.shadowMap.enabled = currentShadowMapEnabled;
-        
+
         // Clean up temporary objects
         geometry.dispose();
         material.dispose();
@@ -159,7 +159,7 @@ export function renderPMREMToEquirect(renderer: WebGLRenderer, pmremTexture: Tex
     renderTarget.texture.mapping = EquirectangularReflectionMapping;
 
     // Log mipmap infos
-    if (debug) console.log('PMREM to Equirect Render Target:', {
+    if (debug) console.log('[MaterialX] PMREM to Equirect Render Target:', {
         width: renderTarget.width,
         height: renderTarget.height,
         mipmaps: renderTarget.texture.mipmaps?.length,

package/src/utils.ts

@@ -2,10 +2,7 @@ import { Context, getParam } from "@needle-tools/engine";
 import { Mesh } from "three";
 
 export const debug = getParam("debugmaterialx");
-
-
-
-
+export const debugUpdate = getParam("debugmaterialxupdate");
 
 /**
  * =====================================
@@ -33,6 +30,44 @@ const patchWebGL2 = () => {
         }
         return uniform4fv.call(this, location, v);
     };
+
+    const uniform3fv = WebGL2RenderingContext.prototype.uniform3fv;
+    WebGL2RenderingContext.prototype.uniform3fv = function (location: WebGLUniformLocation | null, v: Float32Array | number[]) {
+        if (location) {
+            const uniformName = programAndNameToUniformLocation.get(location);
+            if (true) console.log("Calling uniform3fv", { location, v, name: uniformName?.name });
+        }
+        return uniform3fv.call(this, location, v);
+    };
+
+    const uniform3iv = WebGL2RenderingContext.prototype.uniform3iv;
+    WebGL2RenderingContext.prototype.uniform3iv = function (location: WebGLUniformLocation | null, v: Int32Array | number[]) {
+        if (location) {
+            const uniformName = programAndNameToUniformLocation.get(location);
+            if (true) console.log("Calling uniform3iv", { location, v, name: uniformName?.name });
+        }
+        return uniform3iv.call(this, location, v);
+    };
+
+    const uniform3uiv = WebGL2RenderingContext.prototype.uniform3uiv;
+    WebGL2RenderingContext.prototype.uniform3uiv = function (location: WebGLUniformLocation | null, v: Uint32Array | number[]) {
+        if (location) {
+            const uniformName = programAndNameToUniformLocation.get(location);
+            if (true) console.log("Calling uniform3uiv", { location, v, name: uniformName?.name });
+        }
+        return uniform3uiv.call(this, location, v);
+    };
+
+    const uniform3f = WebGL2RenderingContext.prototype.uniform3f;
+    WebGL2RenderingContext.prototype.uniform3f = function (location: WebGLUniformLocation
+        | null, x: number, y: number, z: number) {
+        if (location) {
+            const uniformName = programAndNameToUniformLocation.get(location);
+            if (uniformName?.name !== "diffuse")
+                if (true) console.log("Calling uniform3f", { location, x, y, z, name: uniformName?.name });
+        }
+        return uniform3f.call(this, location, x, y, z);
+    };
 };
 // patchWebGL2();