@needle-tools/gltf-build-pipeline 2.15.3 → 2.15.4

package/dist/transforms/needle_pmrem.d.ts

@@ -10,7 +10,12 @@ export declare class NeedlePmremContext implements INeedleTextureTransformContex
     private available;
     private logger;
     private readonly convertedTextures;
+    /** In-memory dedup: keyed by EXR content hash, resolves to KTX2 bytes */
+    private readonly processingPromises;
     prepare(document: Document): Promise<boolean>;
     process(index: number, texture: Texture, _settings: NEEDLE_compression_texture_schema | null, context: NeedleTransformContext): Promise<boolean>;
+    /** Get or process EXR bytes into KTX2, with in-memory dedup and disk caching */
+    private getOrProcessExr;
+    private processExrToKtx2;
     finalize(document: Document): Promise<void>;
 }

package/dist/transforms/needle_pmrem.js

@@ -4,7 +4,9 @@ import { join, dirname } from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
 import { execFile } from 'child_process';
 import { v4 as uuid } from 'uuid';
+import { createHash } from 'crypto';
 import tmp from 'tmp';
+import { addToCache, tryGetFromCache } from '../cache/index.js';
 /** Resolve the tools/pmrem/ directory relative to this file.
  *  At runtime this file is at dist/transforms/needle_pmrem.js,
  *  so tools/pmrem/ is at ../../tools/pmrem/ */
@@ -46,6 +48,8 @@ export class NeedlePmremContext {
     available = false;
     logger;
     convertedTextures = [];
+    /** In-memory dedup: keyed by EXR content hash, resolves to KTX2 bytes */
+    processingPromises = new Map();
     async prepare(document) {
         this.logger = document.getLogger();
         this.processed = 0;
@@ -97,25 +101,13 @@ export class NeedlePmremContext {
         if (!image)
             return false;
         const name = texture.getName() || `texture_${index}`;
-        const id = uuid();
         const wasExternal = !!texture.getURI();
-        // 1. Run PMREM on the EXR bytes
-        this.logger.info(`pmrem:texture[${index}] ${name}: Running PMREM...`);
         const exrBytes = new Uint8Array(image.buffer, image.byteOffset, image.byteLength);
-        const pmremExrBytes = this.wasmModule.pmrem_exr(exrBytes);
-        // 2. Write PMREM EXR to temp file
-        const tempExr = join(this.tmpDir, `${id}.pmrem.exr`);
-        const tempKtx2 = join(this.tmpDir, `${id}.ktx2`);
-        writeFileSync(tempExr, Buffer.from(pmremExrBytes));
-        // 3. Encode to KTX2 HDR via basisu
-        this.logger.debug(`pmrem:texture[${index}] ${name}: Encoding KTX2 HDR...`);
-        const variant = 'hdr4x4'; // 6x6 needs an update in Needle
-        await encodeKTX2WithBasisu(this.basisuExe, tempExr, tempKtx2, variant, true);
-        if (!existsSync(tempKtx2)) {
-            throw new Error(`pmrem:texture[${index}] ${name}: basisu did not produce output`);
-        }
-        // 4. Read KTX2 and update texture
-        const ktx2Bytes = readFileSync(tempKtx2);
+        // Hash the EXR content for dedup and caching
+        const contentHash = createHash('md5').update(exrBytes).digest('hex');
+        const cacheKey = `pmrem-${contentHash}`;
+        // Get KTX2 bytes — deduplicated and optionally cached
+        const ktx2Bytes = await this.getOrProcessExr(cacheKey, exrBytes, index, name, context.useCache);
         // Remove EXT_texture_exr property since the texture is now KTX2
         const exrProp = texture.getExtension('EXT_texture_exr');
         if (exrProp) {
@@ -157,6 +149,50 @@ export class NeedlePmremContext {
         this.processed++;
         return true;
     }
+    /** Get or process EXR bytes into KTX2, with in-memory dedup and disk caching */
+    getOrProcessExr(cacheKey, exrBytes, index, name, useCache) {
+        // In-memory dedup: if we're already processing identical EXR content, reuse the promise
+        const existing = this.processingPromises.get(cacheKey);
+        if (existing) {
+            this.logger.info(`pmrem:texture[${index}] ${name}: reusing in-flight result (identical content)`);
+            return existing;
+        }
+        const promise = this.processExrToKtx2(cacheKey, exrBytes, index, name, useCache);
+        this.processingPromises.set(cacheKey, promise);
+        return promise;
+    }
+    async processExrToKtx2(cacheKey, exrBytes, index, name, useCache) {
+        // Disk cache check
+        if (useCache) {
+            const cached = tryGetFromCache(cacheKey);
+            if (cached && cached.length > 0) {
+                this.logger.info(`pmrem:texture[${index}] ${name}: loaded from cache`);
+                return cached;
+            }
+        }
+        // 1. Run PMREM on the EXR bytes
+        this.logger.info(`pmrem:texture[${index}] ${name}: Running PMREM...`);
+        const pmremExrBytes = this.wasmModule.pmrem_exr(exrBytes);
+        // 2. Write PMREM EXR to temp file
+        const id = uuid();
+        const tempExr = join(this.tmpDir, `${id}.pmrem.exr`);
+        const tempKtx2 = join(this.tmpDir, `${id}.ktx2`);
+        writeFileSync(tempExr, Buffer.from(pmremExrBytes));
+        // 3. Encode to KTX2 HDR via basisu
+        this.logger.debug(`pmrem:texture[${index}] ${name}: Encoding KTX2 HDR...`);
+        const variant = 'hdr4x4'; // 6x6 needs an update in Needle
+        await encodeKTX2WithBasisu(this.basisuExe, tempExr, tempKtx2, variant, true);
+        if (!existsSync(tempKtx2)) {
+            throw new Error(`pmrem:texture[${index}] ${name}: basisu did not produce output`);
+        }
+        // 4. Read KTX2 result
+        const ktx2Bytes = readFileSync(tempKtx2);
+        // Store in disk cache
+        if (useCache) {
+            addToCache(cacheKey, ktx2Bytes);
+        }
+        return new Uint8Array(ktx2Bytes);
+    }
     async finalize(document) {
         if (this.processed > 0) {
             // Register NEEDLE_pmrem extension and mark converted textures

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.15.3";
+export declare const version = "2.15.4";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.15.3";
+export const version = "2.15.4";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.15.3",
+  "version": "2.15.4",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",