@needle-tools/gltf-build-pipeline 2.15.2 → 2.15.4

package/dist/scripts/pack-gltf.js

@@ -49,7 +49,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             outputFile = getOutputPath(inputFile, outputFile);
         }
         const prevSizeInMB = statSync(inputFile).size / 1024 / 1024;
-        logger.info(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
+        logger.debug(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
         const startTime = Date.now();
         const io = new NodeIO()
             .registerExtensions(ALL_EXTENSIONS)
@@ -123,7 +123,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             }));
         }
         await document.transform(...transforms);
-        logger.info(`← Writing to ${outputFile}`);
+        logger.debug(`← Writing to ${outputFile}`);
         await writeNodeIO(io, outputFile, document);
         if (useCache && cacheKey != undefined && existsSync(outputFile))
             addToCache(cacheKey, readFileSync(outputFile));
@@ -131,7 +131,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         const sizeInMB = statSync(outputFile).size / 1024 / 1024;
         const filename = path.basename(outputFile);
         const gpuMemoryInMB = totalGPUTextureMemory / 1024 / 1024;
-        logger.info(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${outputFile}`);
+        logger.info(`← Compressed '${filename}' ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures) in ${(duration / 1000).toFixed(1)} sec`);
         if (stats) {
             stats.totalFilesProcessed++;
             stats.totalFileSizeInMB += sizeInMB;

package/dist/transforms/needle_mesh_transform.js

@@ -41,7 +41,7 @@ export function determineMeshCompression(document, filePath) {
     }
     // TODO: make this more robus with progressive loading: when progressive loading is enabled we need to run this check on the root mesh and enforce the same mesh compression for all children. Until we can do that we can not use the code for choosing compression based on blendshape size
     if (sizeOfBlendshapes > 0) {
-        document.getLogger().info(`[${NAME}]: will apply meshopt mesh compression because the mesh contains blendshapes`);
+        document.getLogger().debug(`[${NAME}]: will apply meshopt mesh compression because the mesh contains blendshapes`);
         return new NEEDLE_mesh_compression_model_root("meshopt");
     }
     // consider meshopt if blendshape size is > 256 KB
@@ -52,7 +52,7 @@ export function determineMeshCompression(document, filePath) {
     //     /** 25% of the size are blendshapres */
     //     const ratioThreshold = 0.1;
     //     if (blendshapeRatio >= ratioThreshold) {
-    //         document.getLogger().info(`[${NAME}]: will apply meshopt mesh compression because blendshapes are ${(blendshapeRatio * 100).toFixed(2)}% of the mesh size`);
+    //         document.getLogger().debug(`[${NAME}]: will apply meshopt mesh compression because blendshapes are ${(blendshapeRatio * 100).toFixed(2)}% of the mesh size`);
     //         const transform = meshopt({ encoder: MeshoptEncoder });
     //         return transform(document, transform_context);
     //     }
@@ -73,10 +73,10 @@ async function applyMeshCompression(document, transform_context, context) {
         // const rootModel = rootExtension.model as NEEDLE_mesh_compression_model_root;
         if (compressionModel.compression) {
             if (compressionModel.compression === "none") {
-                document.getLogger().info(`[${NAME}]: NONE → will not apply mesh compression`);
+                document.getLogger().debug(`[${NAME}]: NONE → will not apply mesh compression`);
                 return;
             }
-            document.getLogger().info(`[${NAME}]: ${compressionModel.compression.toUpperCase()} → will apply ${compressionModel.compression} mesh compression`);
+            document.getLogger().debug(`[${NAME}]: ${compressionModel.compression.toUpperCase()} → will apply ${compressionModel.compression} mesh compression`);
             if (compressionModel.compression === "draco") {
                 const transform = draco({
                     encodeSpeed: 8,
@@ -92,7 +92,7 @@ async function applyMeshCompression(document, transform_context, context) {
         }
     }
     // by default we still want to run draco if the root doesnt have a compression model configured
-    document.getLogger().info(`[${NAME}]: → will apply draco mesh compression (default)`);
+    document.getLogger().debug(`[${NAME}]: → will apply draco mesh compression (default)`);
     const transform = draco({
         encodeSpeed: 8,
         decodeSpeed: 8,
@@ -103,7 +103,7 @@ async function applyMeshCompression(document, transform_context, context) {
 async function simplifierTransform(document, _context, meshContext) {
     const simplifier = MeshoptSimplifier;
     if (!simplifier) {
-        console.log(`WARN: [${NAME}] simplifier dependency required — install "meshoptimizer".`);
+        document.getLogger().warn(`WARN: [${NAME}] simplifier dependency required — install "meshoptimizer".`);
         return false;
     }
     const logger = document.getLogger();

package/dist/transforms/needle_pmrem.d.ts

@@ -10,7 +10,12 @@ export declare class NeedlePmremContext implements INeedleTextureTransformContex
     private available;
     private logger;
     private readonly convertedTextures;
+    /** In-memory dedup: keyed by EXR content hash, resolves to KTX2 bytes */
+    private readonly processingPromises;
     prepare(document: Document): Promise<boolean>;
     process(index: number, texture: Texture, _settings: NEEDLE_compression_texture_schema | null, context: NeedleTransformContext): Promise<boolean>;
+    /** Get or process EXR bytes into KTX2, with in-memory dedup and disk caching */
+    private getOrProcessExr;
+    private processExrToKtx2;
     finalize(document: Document): Promise<void>;
 }

package/dist/transforms/needle_pmrem.js

@@ -4,7 +4,9 @@ import { join, dirname } from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
 import { execFile } from 'child_process';
 import { v4 as uuid } from 'uuid';
+import { createHash } from 'crypto';
 import tmp from 'tmp';
+import { addToCache, tryGetFromCache } from '../cache/index.js';
 /** Resolve the tools/pmrem/ directory relative to this file.
  *  At runtime this file is at dist/transforms/needle_pmrem.js,
  *  so tools/pmrem/ is at ../../tools/pmrem/ */
@@ -46,6 +48,8 @@ export class NeedlePmremContext {
     available = false;
     logger;
     convertedTextures = [];
+    /** In-memory dedup: keyed by EXR content hash, resolves to KTX2 bytes */
+    processingPromises = new Map();
     async prepare(document) {
         this.logger = document.getLogger();
         this.processed = 0;
@@ -97,25 +101,13 @@ export class NeedlePmremContext {
         if (!image)
             return false;
         const name = texture.getName() || `texture_${index}`;
-        const id = uuid();
         const wasExternal = !!texture.getURI();
-        // 1. Run PMREM on the EXR bytes
-        this.logger.info(`pmrem:texture[${index}] ${name}: Running PMREM...`);
         const exrBytes = new Uint8Array(image.buffer, image.byteOffset, image.byteLength);
-        const pmremExrBytes = this.wasmModule.pmrem_exr(exrBytes);
-        // 2. Write PMREM EXR to temp file
-        const tempExr = join(this.tmpDir, `${id}.pmrem.exr`);
-        const tempKtx2 = join(this.tmpDir, `${id}.ktx2`);
-        writeFileSync(tempExr, Buffer.from(pmremExrBytes));
-        // 3. Encode to KTX2 HDR via basisu
-        this.logger.debug(`pmrem:texture[${index}] ${name}: Encoding KTX2 HDR...`);
-        const variant = 'hdr4x4'; // 6x6 needs an update in Needle
-        await encodeKTX2WithBasisu(this.basisuExe, tempExr, tempKtx2, variant, true);
-        if (!existsSync(tempKtx2)) {
-            throw new Error(`pmrem:texture[${index}] ${name}: basisu did not produce output`);
-        }
-        // 4. Read KTX2 and update texture
-        const ktx2Bytes = readFileSync(tempKtx2);
+        // Hash the EXR content for dedup and caching
+        const contentHash = createHash('md5').update(exrBytes).digest('hex');
+        const cacheKey = `pmrem-${contentHash}`;
+        // Get KTX2 bytes — deduplicated and optionally cached
+        const ktx2Bytes = await this.getOrProcessExr(cacheKey, exrBytes, index, name, context.useCache);
         // Remove EXT_texture_exr property since the texture is now KTX2
         const exrProp = texture.getExtension('EXT_texture_exr');
         if (exrProp) {
@@ -157,6 +149,50 @@ export class NeedlePmremContext {
         this.processed++;
         return true;
     }
+    /** Get or process EXR bytes into KTX2, with in-memory dedup and disk caching */
+    getOrProcessExr(cacheKey, exrBytes, index, name, useCache) {
+        // In-memory dedup: if we're already processing identical EXR content, reuse the promise
+        const existing = this.processingPromises.get(cacheKey);
+        if (existing) {
+            this.logger.info(`pmrem:texture[${index}] ${name}: reusing in-flight result (identical content)`);
+            return existing;
+        }
+        const promise = this.processExrToKtx2(cacheKey, exrBytes, index, name, useCache);
+        this.processingPromises.set(cacheKey, promise);
+        return promise;
+    }
+    async processExrToKtx2(cacheKey, exrBytes, index, name, useCache) {
+        // Disk cache check
+        if (useCache) {
+            const cached = tryGetFromCache(cacheKey);
+            if (cached && cached.length > 0) {
+                this.logger.info(`pmrem:texture[${index}] ${name}: loaded from cache`);
+                return cached;
+            }
+        }
+        // 1. Run PMREM on the EXR bytes
+        this.logger.info(`pmrem:texture[${index}] ${name}: Running PMREM...`);
+        const pmremExrBytes = this.wasmModule.pmrem_exr(exrBytes);
+        // 2. Write PMREM EXR to temp file
+        const id = uuid();
+        const tempExr = join(this.tmpDir, `${id}.pmrem.exr`);
+        const tempKtx2 = join(this.tmpDir, `${id}.ktx2`);
+        writeFileSync(tempExr, Buffer.from(pmremExrBytes));
+        // 3. Encode to KTX2 HDR via basisu
+        this.logger.debug(`pmrem:texture[${index}] ${name}: Encoding KTX2 HDR...`);
+        const variant = 'hdr4x4'; // 6x6 needs an update in Needle
+        await encodeKTX2WithBasisu(this.basisuExe, tempExr, tempKtx2, variant, true);
+        if (!existsSync(tempKtx2)) {
+            throw new Error(`pmrem:texture[${index}] ${name}: basisu did not produce output`);
+        }
+        // 4. Read KTX2 result
+        const ktx2Bytes = readFileSync(tempKtx2);
+        // Store in disk cache
+        if (useCache) {
+            addToCache(cacheKey, ktx2Bytes);
+        }
+        return new Uint8Array(ktx2Bytes);
+    }
     async finalize(document) {
         if (this.processed > 0) {
             // Register NEEDLE_pmrem extension and mark converted textures

package/dist/transforms/needle_progressive.js

@@ -213,7 +213,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${textureIndex}`);
                 continue;
             }
-            const mimeTypeResult = validateImageMimeType(texture.getMimeType(), texture.getImage());
+            const mimeTypeResult = validateImageMimeType(texture.getMimeType(), texture.getImage(), logger);
             if (mimeTypeResult?.error) {
                 logger.warn(`[${NAME}] Texture[${index}] has invalid mime type! Reported \"${texture.getMimeType()}\" but is actually \"${mimeTypeResult.mimeType}\": auto-fix`);
                 texture.setMimeType(mimeTypeResult.mimeType);
@@ -514,7 +514,7 @@ export async function resizeTextureToMax(texture, maxSize, logger) {
     await resizeTexture(options, texture, logger);
 }
 async function resizeTexture(options, texture, logger, uri, slots) {
-    const mimeTypeResult = validateImageMimeType(texture.getMimeType(), texture.getImage());
+    const mimeTypeResult = validateImageMimeType(texture.getMimeType(), texture.getImage(), logger);
     if (mimeTypeResult?.error) {
         logger.warn(`[${NAME}] Texture[${uri}] has invalid mime type: ${mimeTypeResult.error}: auto-fix`);
         texture.setMimeType(mimeTypeResult.mimeType);

package/dist/transforms/needle_texture_transform.js

@@ -88,7 +88,7 @@ export const needle_texture_transform = function (options) {
     });
 };
 async function processTexture(texture, index, document, logger, context) {
-    const validationResult = validateImageMimeType(texture.getMimeType(), texture.getImage());
+    const validationResult = validateImageMimeType(texture.getMimeType(), texture.getImage(), logger);
     if (validationResult?.error) {
         logger.warn(`WARN: Texture ${texture.getName()} [${index}] has an incorrect mime type \"${texture.getMimeType()}\" → auto-correcting to ${validationResult.mimeType}`);
         texture.setMimeType(validationResult.mimeType);

package/dist/utils/validate.d.ts

@@ -1,4 +1,5 @@
-export declare function validateImageMimeType(mimetype: string, image: Uint8Array): {
+import { ILogger } from "@gltf-transform/core";
+export declare function validateImageMimeType(mimetype: string, image: Uint8Array, logger: ILogger): {
     error: boolean;
     message: string;
     mimeType: string;

package/dist/utils/validate.js

@@ -1,9 +1,9 @@
 import { ImageUtils } from "@gltf-transform/core";
-export function validateImageMimeType(mimetype, image) {
+export function validateImageMimeType(mimetype, image, logger) {
     const actualMimeType = ImageUtils.getMimeType(image);
     if (actualMimeType != null && actualMimeType != mimetype) {
         const msg = `Image reported wrong mime type \"${mimetype}\" but is actually \"${actualMimeType}\"`;
-        console.error(`WARN: ${msg}`);
+        logger.warn(msg);
         return {
             error: true,
             message: msg,
@@ -11,7 +11,7 @@ export function validateImageMimeType(mimetype, image) {
         };
     }
     else if (actualMimeType === null) {
-        console.warn(`WARN: Could not validate image type: \"${mimetype}\"`);
+        logger.debug(`Could not validate image type: \"${mimetype}\"`);
     }
     return null;
 }

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.15.2";
+export declare const version = "2.15.4";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.15.2";
+export const version = "2.15.4";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.15.2",
+  "version": "2.15.4",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",