@needle-tools/gltf-build-pipeline 2.15.0-next.fd1fff4 → 2.15.0

package/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.15.0] - 2026-03-20
+- add: EXR environment map compression (PMREM / FASTHDR) for Needle Engine, supporting both Unity and Blender exports
+- fix: cache hashing stability (circular reference handling)
+- fix: progressive texture GUID generation now based on texture content hash
+
 ## [2.14.0] - 2026-01-30
 - add: KHR_materials_pbrSpecularGlossiness extension support
 - fix: reduce `resample` tolerance to `1e-6` to avoid animation issues with some assets

package/dist/cli/index.js

@@ -10,6 +10,7 @@ import { existsSync } from 'fs';
 import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
 import path, { resolve } from 'path';
 import { ERROR_CODES } from '../constants.js';
+import { trackPipelineStart, trackPipelineEnd, trackError } from '../utils/analytics.js';
 // For testing / dev you can run `npm link` in the package directory
 // For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
 program
@@ -73,6 +74,8 @@ Each version will be compressed and written to the output directory.
         return;
     }
     logger.info(`[Needle Build Pipeline] v${getVersion()} — Transform '${args.input}'`);
+    const pipelineMode = [options.progressive && 'progressive', options.compress && 'compress'].filter(Boolean).join('+');
+    trackPipelineStart({ mode: pipelineMode, usecase: options.usecase?.toString() });
     const stats = {
         startTime: Date.now(),
         totalGPUMemoryInMB: 0,
@@ -103,6 +106,7 @@ Each version will be compressed and written to the output directory.
     const outputIsDirectory = output && isDirectory(output);
     if (inputIsDirectory && output && !outputIsDirectory) {
         logger.error("When <input> is a directory then <output> must also be a directory");
+        trackError(new Error('Invalid args: input is directory but output is not'), { mode: pipelineMode });
         process.exit(ERROR_CODES.INVALID_ARGS);
     }
     // Handle the output bath being relative to the input path
@@ -126,6 +130,7 @@ Each version will be compressed and written to the output directory.
             }
             else {
                 logger.error(`ERR: Previously found file is now missing: \"${file}\"`);
+                trackError(new Error(`Lost file: ${file}`), { mode: pipelineMode, usecase: options.usecase?.toString(), file });
                 process.exit(ERROR_CODES.LOST_FILE);
             }
         }
@@ -148,8 +153,10 @@ Each version will be compressed and written to the output directory.
                 logger,
             }).catch(err => {
                 logger.error(err);
+                trackError(err, { mode: pipelineMode, usecase: options.usecase?.toString(), file });
                 process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
             }) === false) {
+                trackError(new Error('Progressive processing returned false'), { mode: pipelineMode, usecase: options.usecase?.toString(), file });
                 process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
             }
             if (verbose) {
@@ -170,6 +177,7 @@ Each version will be compressed and written to the output directory.
             if (progressive_results.length > 0) {
                 for (const res of progressive_results) {
                     if (await packGLTF(res, res, opts) === false) {
+                        trackError(new Error('Packing failed'), { mode: pipelineMode, usecase: options.usecase?.toString(), file: res });
                         process.exit(ERROR_CODES.PACKING_FAILED);
                     }
                 }
@@ -178,12 +186,22 @@ Each version will be compressed and written to the output directory.
             // We can not just check if the file is in the list of results because the output path might have changed the name
             else {
                 if (await packGLTF(file, output, opts) === false) {
+                    trackError(new Error('Packing failed'), { mode: pipelineMode, usecase: options.usecase?.toString(), file });
                     process.exit(ERROR_CODES.PACKING_FAILED);
                 }
             }
         }
     });
     printStats(stats, logger, config);
+    trackPipelineEnd({
+        mode: pipelineMode,
+        usecase: config?.usecase?.toString(),
+        filesProcessed: stats.totalFilesProcessed,
+        sizeBefore_MB: stats.totalFileSizeInMBBefore,
+        sizeAfter_MB: stats.totalFileSizeInMB,
+        gpuMemory_MB: stats.totalGPUMemoryInMB,
+        duration_ms: Date.now() - stats.startTime,
+    });
     if (useCache)
         limitCacheSize();
 });
@@ -194,5 +212,6 @@ program.run().catch(err => {
         // ignore
         return;
     }
+    trackError(err);
     throw err;
 });

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts

@@ -23,5 +23,6 @@ export interface IExtensionWriter {
     write(context: WriterContext, prop: IExtensibleProperty): void;
 }
 export declare let currentGenerator: string;
+export declare function setOpaqueInputFile(path: string): void;
 export declare function createOpaqueExtension(name: string, types: PropertyType | PropertyType[], opts?: Options): IOpaqueExtension;
 export {};

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -3,11 +3,16 @@ import { getTexture, tryGetInArray, writeExtension } from '../utils.js';
 import { isLightmap } from '../../utils/texture.js';
 import { NEEDLE_progressive_texture_settings } from '../NEEDLE_progressive_texture_settings/index.js';
 import { testAssert } from '../../utils/test-assert.js';
+import { existsSync, readFileSync } from 'fs';
+import { dirname, join } from 'path';
 const ALL_PROPERTY_TYPES = [];
 for (const key in PropertyType) {
     ALL_PROPERTY_TYPES.push(PropertyType[key]);
 }
 export let currentGenerator = "";
+/** Set before io.read() to allow resolving external file paths (e.g. .exr) in component data */
+let _opaqueInputFile = "";
+export function setOpaqueInputFile(path) { _opaqueInputFile = path; }
 // an opaque extension can be used to pass data through the gltf transform process
 // it should also be able to re-write/fix json pointers after the transformation which might have shuffled them around 
 export function createOpaqueExtension(name, types, opts) {
@@ -539,6 +544,10 @@ class JsonPointerHandler {
     debug = false;
     breakPoint = false;
     VRM_0_MaterialIndexRegex = new RegExp(/materialProperties\/(?<material_index>\d+)\//g);
+    /** Cache of external EXR files already injected as Textures (keyed by filename) */
+    externalExrTextures = new Map();
+    /** External EXR references that need URI resolution at write time */
+    externalExrPointers = [];
     constructor(document, extension_name) {
         this.document = document;
         this.extension_name = extension_name;
@@ -620,6 +629,37 @@ class JsonPointerHandler {
                         else
                             console.warn("WARN: failed registering pointer", fullPath, value);
                     }
+                    // External EXR file reference (e.g. "StudioHDRI_ferndale_studio_04_1k.exr" in a ReflectionProbe)
+                    // Inject the file into the document as a Texture so PMREM can process it
+                    if (!isTexturePointer && value.match(/\.exr$/i)) {
+                        const inputDir = _opaqueInputFile ? dirname(_opaqueInputFile) : '';
+                        const exrPath = inputDir ? join(inputDir, value) : '';
+                        if (exrPath && existsSync(exrPath)) {
+                            // Check if we already injected this file
+                            let texture = this.externalExrTextures.get(value);
+                            if (!texture) {
+                                const exrBytes = readFileSync(exrPath);
+                                texture = this.document.createTexture(value)
+                                    .setImage(new Uint8Array(exrBytes))
+                                    .setMimeType('image/exr')
+                                    .setURI(value);
+                                this.externalExrTextures.set(value, texture);
+                                if (this.debug)
+                                    console.log(`Injected external EXR "${value}" as Texture`);
+                            }
+                            // Set a reference so gltf-transform doesn't prune it
+                            prop.setReference(`exr_${value}`, texture, { channels: TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A });
+                            const buffer = this.document.getRoot().listBuffers()[0];
+                            if (buffer)
+                                prop.setReference(`exr_buffer_${value}`, buffer);
+                            // At write time, update the property to the texture's current URI
+                            // (PMREM will have updated it from "foo.exr" to "foo.pmrem.ktx2")
+                            this.externalExrPointers.push({ obj, key, texture });
+                        }
+                        else if (exrPath) {
+                            console.warn(`WARN: External EXR not found: ${exrPath}`);
+                        }
+                    }
                     if (value.startsWith("/materials/")) {
                         if (this.debug)
                             console.log("Read pointer", value);
@@ -777,6 +817,25 @@ class JsonPointerHandler {
             if (ptr.resolve(context, step))
                 ptr.write();
         });
+        // Resolve external EXR pointers: update component data with the texture's current URI
+        for (const { obj, key, texture } of this.externalExrPointers) {
+            const uri = texture.getURI() || texture.getName() || '';
+            if (uri) {
+                if (this.debug)
+                    console.log(`< Resolved external EXR: ${obj[key]} → ${uri}`);
+                obj[key] = uri;
+                // Ensure the image definition in the output JSON has the URI set.
+                // When PMREM clears image data (setImage(null)) for external textures,
+                // gltf-transform's core writer won't set a uri or bufferView on the image.
+                const imageIndex = context.imageIndexMap.get(texture);
+                if (imageIndex !== undefined) {
+                    const images = context.jsonDoc.json.images;
+                    if (images && images[imageIndex] && !images[imageIndex].uri) {
+                        images[imageIndex].uri = uri;
+                    }
+                }
+            }
+        }
     }
 }
 class JsonPointer {

package/dist/scripts/pack-gltf.js

@@ -5,7 +5,7 @@ import draco3d from 'draco3dgltf';
 import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 import { dedup, metalRough, prune, resample } from '@gltf-transform/functions';
-import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, } from '../extensions/index.js';
+import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, setOpaqueInputFile, } from '../extensions/index.js';
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
 import { getOutputPath, getVersion, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
@@ -66,8 +66,9 @@ export async function packGLTF(inputFile, outputFile, options) {
             'meshopt.decoder': MeshoptDecoder,
             'meshopt.encoder': MeshoptEncoder,
         });
-        // Set input file path so NEEDLE_lightmaps can resolve external EXR files during preread
+        // Set input file path so extensions can resolve external EXR files
         NEEDLE_lightmaps_ext.inputFile = inputFile;
+        setOpaqueInputFile(inputFile);
         const document = await ioTryReadWithMissingResources(io, inputFile);
         if (options.logger) {
             document.setLogger(options.logger);

package/dist/utils/analytics.d.ts

@@ -0,0 +1,24 @@
+export interface PipelineAnalyticsInfo {
+    /** 'compress' | 'progressive' | 'compress+progressive' */
+    mode: string;
+    usecase?: string;
+    filesProcessed: number;
+    sizeBefore_MB: number;
+    sizeAfter_MB: number;
+    gpuMemory_MB: number;
+    duration_ms: number;
+    extensionsUsed?: string[];
+}
+/** Send a "pipeline_start" event. */
+export declare function trackPipelineStart(info: {
+    mode: string;
+    usecase?: string;
+}): void;
+/** Send a pipeline error event. Deduplicates identical errors within 1 minute. */
+export declare function trackError(error: unknown, context?: {
+    mode?: string;
+    usecase?: string;
+    file?: string;
+}): void;
+/** Send a "pipeline_end" event with size reduction and extension info. */
+export declare function trackPipelineEnd(info: PipelineAnalyticsInfo): void;

package/dist/utils/analytics.js

@@ -0,0 +1,126 @@
+import os from 'os';
+import https from 'https';
+import { getVersion } from './version.js';
+const RYBBIT_URL = 'https://needle.tools/api/v1/rum/t';
+const SITE_ID = '5c44f60eb2d7';
+/** Build a User-Agent string that UA parsers recognize as desktop + correct OS. */
+function getUserAgent() {
+    const v = getVersion();
+    const platform = os.platform();
+    const release = os.release();
+    const arch = os.arch();
+    const node = process.version;
+    // Map Node's os.platform() to a UA-parser-friendly OS token
+    let osToken;
+    if (platform === 'darwin') {
+        // macOS: "Macintosh; Intel Mac OS X 25_3_0"
+        const ver = release.replace(/\./g, '_');
+        osToken = `Macintosh; Intel Mac OS X ${ver}`;
+    }
+    else if (platform === 'win32') {
+        // Windows: "Windows NT 10.0; Win64; x64"
+        osToken = `Windows NT ${release}; Win64; ${arch}`;
+    }
+    else {
+        // Linux / other: "X11; Linux x86_64"
+        osToken = `X11; Linux ${arch}`;
+    }
+    return `Mozilla/5.0 (${osToken}) NeedleBuildPipeline/${v} Node/${node}`;
+}
+/** Fire-and-forget analytics event. Never throws, never blocks. */
+function sendEvent(eventName, properties, type = 'custom_event') {
+    try {
+        const body = JSON.stringify({
+            site_id: SITE_ID,
+            type,
+            event_name: eventName,
+            properties: JSON.stringify(properties),
+            user_agent: getUserAgent(),
+        });
+        const url = new URL(RYBBIT_URL);
+        const req = https.request({
+            hostname: url.hostname,
+            path: url.pathname,
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Content-Length': Buffer.byteLength(body),
+            },
+            timeout: 5000,
+        });
+        req.on('error', () => { });
+        req.on('timeout', () => { req.destroy(); });
+        req.end(body);
+    }
+    catch {
+        // silent
+    }
+}
+/** Send a "pipeline_start" event. */
+export function trackPipelineStart(info) {
+    sendEvent('pipeline_start', {
+        version: getVersion(),
+        mode: info.mode,
+        usecase: info.usecase,
+        platform: process.platform,
+        arch: process.arch,
+        node: process.version,
+    });
+}
+/** Dedup cache to avoid logging the same error repeatedly in one session */
+const recentErrors = new Map();
+const ERROR_DEDUP_MS = 60_000;
+/** Send a pipeline error event. Deduplicates identical errors within 1 minute. */
+export function trackError(error, context) {
+    const err = error instanceof Error ? error : new Error(String(error));
+    const message = (err.message || 'Unknown error').substring(0, 500);
+    const stack = (err.stack || '').substring(0, 2000);
+    const errorName = err.name || 'Error';
+    // Dedup
+    const key = `${errorName}|${message}`;
+    const now = Date.now();
+    const cached = recentErrors.get(key);
+    if (cached && now - cached.time < ERROR_DEDUP_MS) {
+        cached.count++;
+        return;
+    }
+    recentErrors.set(key, { time: now, count: (cached?.count || 0) + 1 });
+    const properties = {
+        version: getVersion(),
+        error_name: errorName + (cached && cached.count > 1 ? ` x${cached.count}` : ''),
+        message,
+        platform: process.platform,
+        arch: process.arch,
+        node: process.version,
+    };
+    if (stack)
+        properties.stack = stack;
+    if (context?.mode)
+        properties.mode = context.mode;
+    if (context?.usecase)
+        properties.usecase = context.usecase;
+    if (context?.file)
+        properties.file = context.file;
+    sendEvent(errorName, properties, 'error');
+}
+/** Send a "pipeline_end" event with size reduction and extension info. */
+export function trackPipelineEnd(info) {
+    const reductionPct = info.sizeBefore_MB > 0
+        ? ((1 - info.sizeAfter_MB / info.sizeBefore_MB) * 100).toFixed(1)
+        : '0';
+    sendEvent('pipeline_end', {
+        version: getVersion(),
+        mode: info.mode,
+        usecase: info.usecase,
+        files: info.filesProcessed,
+        size_before_mb: +info.sizeBefore_MB.toFixed(2),
+        size_after_mb: +info.sizeAfter_MB.toFixed(2),
+        reduction_pct: +reductionPct,
+        gpu_memory_mb: +info.gpuMemory_MB.toFixed(2),
+        duration_ms: info.duration_ms,
+        extensions: info.extensionsUsed?.join(','),
+        platform: process.platform,
+        arch: process.arch,
+        node: process.version,
+    });
+}

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.15.0-next.fd1fff4";
+export declare const version = "2.15.0";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.15.0-next.fd1fff4";
+export const version = "2.15.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.15.0-next.fd1fff4",
+  "version": "2.15.0",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",

package/tools/pmrem/basis/linux/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt

@@ -0,0 +1 @@
+d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73

package/tools/pmrem/basis/osx/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt

@@ -0,0 +1 @@
+d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73

package/tools/pmrem/basis/wasm/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt

@@ -0,0 +1 @@
+d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73