@needle-tools/gltf-build-pipeline 2.14.0 → 2.15.0-next.0c5bc39

package/dist/cache/cache.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" resolution-mode="require"/>
 import { ExtensibleProperty } from "@gltf-transform/core";
 import { Logger } from "@donmccurdy/caporal";
 /**

package/dist/cache/cache.js

@@ -5,9 +5,9 @@ import path from 'path';
 import { getVersion } from "../utils/version.js";
 // import * as CHECKDISCSPACE from 'check-disk-space'
 import xxhash from "xxhash-wasm";
-let hasher = null;
+let xxHashModule = null;
 xxhash().then(module => {
-    hasher = module.create32();
+    xxHashModule = module;
 });
 /**
  * Get the available space on the cache directory in bytes.
@@ -304,9 +304,9 @@ function hashObject(obj, depth = 0) {
     }
 }
 function hashBuffer(buffer, offset, length) {
-    if (hasher) {
+    if (xxHashModule) {
         const view = new Uint8Array(buffer, offset, length);
-        return hasher.update(view).digest();
+        return xxHashModule.h32Raw(view);
     }
     let hash = 0;
     const view = new Uint8Array(buffer, offset, length);

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -1,6 +1,8 @@
 import { Extension, ExtensionProperty, PropertyType, Node, TextureChannel } from '@gltf-transform/core';
 import { getTexture, tryGetInArray, writeExtension } from '../utils.js';
 import { isLightmap } from '../../utils/texture.js';
+import { NEEDLE_progressive_texture_settings } from '../NEEDLE_progressive_texture_settings/index.js';
+import { testAssert } from '../../utils/test-assert.js';
 const ALL_PROPERTY_TYPES = [];
 for (const key in PropertyType) {
     ALL_PROPERTY_TYPES.push(PropertyType[key]);
@@ -123,8 +125,11 @@ export function createOpaqueExtension(name, types, opts) {
                             const prop = this.createExtension();
                             this.jsonPointerHandler.read(context, texture, ext, prop);
                             texture.setExtension(EXTENSION_NAME, prop);
-                            const handler = new TextureHandler(this.document, textureDefinition, EXTENSION_NAME, ext, debugLog);
+                            const handler = new TextureHandler(this.document, textureDefinition, EXTENSION_NAME, ext, debugLog, texture);
                             prop.add(handler);
+                            const sourceIndex = textureDefinition.source;
+                            if (debugLog)
+                                console.log("Found texture with extension", EXTENSION_NAME, "at index", index, "source image index", sourceIndex, "name", texture.getName() || "<unnamed>");
                         });
                         break;
                     case PropertyType.NODE:
@@ -188,11 +193,11 @@ export function createOpaqueExtension(name, types, opts) {
             const textures = this.document.getRoot().listTextures();
             for (let i = 0; i < textures.length; i++) {
                 const tex = textures[i];
-                const texDef = textureDefinitions[i];
-                const name = texDef?.name;
                 if (isLightmap(tex) && tex.getMimeType() !== "image/exr") {
                     prop.setReference("lightmapTexture", tex, { channels: TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A });
-                    logger.info(`Found lightmap texture ${name}: ${tex.getMimeType()} in ${EXTENSION_NAME}`);
+                    const texDef = textureDefinitions[i]; // @TODO: textureDefinitions and textures might not be in the same order - we need to find the texture by source index instead of assuming the order is the same
+                    const name = tex.getName() || texDef?.name;
+                    logger.info(`Found lightmap texture \"${name}\" (${tex.getMimeType()}) in ${EXTENSION_NAME}`);
                 }
             }
         }
@@ -389,39 +394,48 @@ class GenericWriter {
 class TextureHandler {
     document;
     textureDefinition;
-    index;
     key;
     ext;
     debug = false;
+    /** Reference to the gltf-transform Texture object, used to resolve the current image index at write time via imageIndexMap */
+    textureRef;
     assignTo(obj) {
         Object.assign(obj, this.ext);
     }
-    constructor(doc, def, key, ext, debug) {
+    constructor(doc, def, key, ext, debug, textureRef) {
         this.document = doc;
         this.textureDefinition = def;
         this.key = key;
         this.ext = ext;
         this.debug = debug;
-        this.index = this.tryGetSourceIndex(this.textureDefinition);
-        if (debug)
-            console.log(def);
+        this.textureRef = textureRef;
+        if (debug) {
+            const index = this.tryGetSourceIndex(this.textureDefinition);
+            console.log(index, this.textureDefinition);
+        }
     }
     write(ctx) {
-        let foundTexture = false;
-        const textureDefs = ctx.jsonDoc.json.textures || [];
-        const imageIndex = this.index !== undefined ? this.index : this.tryGetSourceIndex(this.textureDefinition);
+        // Resolve the current image index from the gltf-transform graph reference.
+        // This is stable even after dedup/prune operations that re-index images.
+        const imageIndex = ctx.imageIndexMap.get(this.textureRef);
+        testAssert(imageIndex !== undefined || this.key === "EXT_texture_exr", `imageIndexMap missing texture ref for "${this.key}": ${this.textureRef.getName() || "<unnamed>"}`);
         if (imageIndex === undefined) {
-            console.warn("WARN: no source index found for texture", this.key);
+            if (this.debug) {
+                console.warn("WARN: failed to resolve image index for texture via imageIndexMap:", this.key, this.textureRef.getName() || "<unnamed>");
+            }
+            return;
         }
+        let foundTexture = false;
+        const textureDefs = ctx.jsonDoc.json.textures || [];
         textureDefs?.forEach((textureDef, i) => {
             if (foundTexture)
                 return;
             const textureImageIndex = this.tryGetSourceIndex(textureDef);
             if (imageIndex === textureImageIndex) {
-                if (!textureDef.extensions || !textureDef.extensions[this.key]) {
-                    // assign texture extension again
+                foundTexture = true;
+                if (!textureDef.extensions?.[this.key]) {
+                    // assign texture extension
                     textureDef.extensions = textureDef.extensions || {};
-                    foundTexture = true;
                     if (this.debug)
                         console.log("Re-Assign extension:", this.key, "at", i);
                     const newExt = { ...this.ext };
@@ -429,20 +443,16 @@ class TextureHandler {
                         newExt.source = imageIndex;
                     textureDef.extensions[this.key] = newExt;
                 }
+                // else: extension already written by another handler for the same image
+                // (happens when gltf-transform dedup merges texture defs that shared the same source)
             }
         });
+        testAssert(foundTexture || this.key === "EXT_texture_exr", `failed to match texture for extension "${this.key}", imageIndex=${imageIndex}, textures=${textureDefs.length}, name=${this.textureRef.getName() || "<unnamed>"}, type=${this.textureRef.getMimeType()}`);
         if (foundTexture === false) {
-            if (this.key !== "EXT_texture_exr" && this.debug)
-                console.warn("WARN: failed to re-assign extension:", this.key, this.index, this.tryGetSourceIndex(this.textureDefinition), textureDefs.length);
-        }
-        // this.document
-        //     .getRoot()
-        //     .listTextures()
-        //     .forEach((texture, index) => {
-        //         if (foundTexture) return;
-        //         if (!texture.getExtension(this.key)) return;
-        //         const imageIndex = ctx.imageIndexMap.get(texture);
-        //     });
+            if (this.debug) {
+                console.warn("WARN: failed to re-assign extension:", this.key, "imageIndex", imageIndex, textureDefs.length, this.textureRef.getName() || "<unnamed>", this.textureDefinition);
+            }
+        }
     }
     tryGetSourceIndex(def) {
         let index = def.source;
@@ -857,7 +867,8 @@ class TextureIndexResolver extends PointerResolver {
     resolve(context) {
         const imgIndex = context.imageIndexMap.get(this.inputObject);
         if (imgIndex === undefined) {
-            console.error("ERR: image is missing", imgIndex);
+            testAssert(false, `imageIndexMap missing texture: ${this.inputObject.getName() || "<unnamed>"}`);
+            console.error("ERR: image is missing");
             return -1;
         }
         // ensure we have a texture array
@@ -878,11 +889,18 @@ class TextureIndexResolver extends PointerResolver {
         // create a new texture def entry
         if (this.debug)
             console.log("Could not find texture, creating a new one: " + this._debugName);
-        const hasExtensionWithSource = this.inputDefinition.extensions && Object.keys(this.inputDefinition.extensions).some(key => this.inputDefinition.extensions[key]["source"] !== undefined);
+        // Remove settings-only extensions that are consumed during processing and should not appear in the output.
+        let extensions = this.inputDefinition.extensions ? { ...this.inputDefinition.extensions } : undefined;
+        if (extensions) {
+            delete extensions[NEEDLE_progressive_texture_settings.EXTENSION_NAME];
+            if (Object.keys(extensions).length === 0)
+                extensions = undefined;
+        }
+        const hasExtensionWithSource = extensions && Object.keys(extensions).some(key => extensions[key]?.["source"] !== undefined);
         const textureDef = {
             source: hasExtensionWithSource ? undefined : imgIndex,
             sampler: this.findOrCreateSampler(context.jsonDoc.json),
-            extensions: this.inputDefinition.extensions,
+            extensions: extensions,
             extras: this.inputDefinition.extras,
         };
         context.jsonDoc.json.textures.push(textureDef);

package/dist/scripts/pack-gltf.js

@@ -9,6 +9,7 @@ import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
 import { getOutputPath, getVersion, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
+import { TestAssertionError } from '../utils/test-assert.js';
 /** Compress a glTF file
  */
 export async function packGLTF(inputFile, outputFile, options) {
@@ -64,8 +65,13 @@ export async function packGLTF(inputFile, outputFile, options) {
             'meshopt.encoder': MeshoptEncoder,
         });
         const document = await ioTryReadWithMissingResources(io, inputFile); // await io.read(inputFile);
-        /** @ts-ignore */
-        document.getLogger().verbosity = options.verbose ? Verbosity.DEBUG : options.debug ? Verbosity.INFO : Verbosity.WARN;
+        if (options.logger) {
+            document.setLogger(options.logger);
+        }
+        else {
+            /** @ts-ignore */
+            document.getLogger().verbosity = options.verbose ? Verbosity.DEBUG : options.debug ? Verbosity.INFO : Verbosity.WARN;
+        }
         let totalGPUTextureMemory = 0;
         /** @type {Array<import('@gltf-transform/core').Transform>} */
         const transforms = [
@@ -128,7 +134,10 @@ export async function packGLTF(inputFile, outputFile, options) {
         return true;
     }
     catch (err) {
-        logger.error(`Packing of "${path.basename(inputFile)}" using version ${getVersion()} failed with error:\n${err.message}`);
+        if (err instanceof TestAssertionError)
+            throw err;
+        const shortedStackTrace = err instanceof Error && err.stack ? err.stack.split('\n').slice(0, 2).join('\n') : '';
+        logger.error(`Packing of "${path.basename(inputFile)}" using version ${getVersion()} failed with error:\n${err.message}\n${shortedStackTrace}`);
         if (debug)
             throw err;
         return false;

package/dist/transforms/needle_progressive.js

@@ -13,6 +13,7 @@ import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptEncoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive, determineMeshCompression } from './needle_mesh_transform.js';
 import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
+import { TestAssertionError } from '../utils/test-assert.js';
 export async function make_progressive(opts) {
     const file = opts.path;
     let success = true;
@@ -78,9 +79,14 @@ async function onProcess(file, opts) {
         });
         let document = await ioTryReadWithMissingResources(io, file); // await io.read(file);
         if (document) {
-            const logger = document.getLogger();
-            //@ts-ignore
-            logger.verbosity = (opts.debug || opts.verbose) ? Verbosity.DEBUG : Verbosity.WARN;
+            if (opts.logger) {
+                document.setLogger(opts.logger);
+            }
+            else {
+                //@ts-ignore
+                document.getLogger().verbosity = (opts.debug || opts.verbose) ? Verbosity.DEBUG : Verbosity.WARN;
+            }
+            const logger = opts.logger || document.getLogger();
             let smallesTextureSize = 128;
             if (opts.config.usecase === "product") {
                 smallesTextureSize = 512;
@@ -120,6 +126,8 @@ async function onProcess(file, opts) {
         }
     }
     catch (err) {
+        if (err instanceof TestAssertionError)
+            throw err;
         console.log("ERR: failed to process file \"" + file + "\", Reason: " + err.message, "\nStack:\n" + err.stack);
         return false;
     }
@@ -151,6 +159,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             return Promise.resolve();
         }
         let index = 0;
+        const textureHashCache = new Map();
         for (const texture of doc.getRoot().listTextures()) {
             const textureIndex = index++;
             const name = texture.getName();
@@ -177,8 +186,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                     continue;
                 }
                 else {
+                    if (!textureHashCache.has(texture))
+                        textureHashCache.set(texture, getHash(texture));
                     settings = {
-                        guid: generateGuid(filename + "_texture_" + index.toString()),
+                        guid: generateGuid(textureHashCache.get(texture)),
                         maxSize: _options.size[0],
                     };
                     logger.debug(`[${NAME}] No settings found for texture[${index}] - will use default settings (maxSize: ${settings.maxSize}, guid: ${settings.guid})`);
@@ -213,7 +224,13 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 logger.debug(`[${NAME}] Skipping, texture[${textureIndex}] is already smaller than the requested size: ${currentSize}px <= ${settings.maxSize}px`);
                 continue;
             }
-            const textureGuid = settings.guid?.length ? settings.guid : generateGuid(filename + "_texture_" + index.toString());
+            if (!settings.guid?.length) {
+                if (!textureHashCache.has(texture))
+                    textureHashCache.set(texture, getHash(texture));
+                const hash = textureHashCache.get(texture);
+                settings.guid = generateGuid(hash);
+            }
+            const textureGuid = settings.guid;
             /** This is the size of the texture that is embedded in the glTF. No LOD should be smaller than this */
             const maxSize = settings.maxSize;
             const LOD_LEVEL = new Array();
@@ -360,7 +377,7 @@ async function createTextureLod(args, opts) {
                     // see https://linear.app/needle/issue/NE-4985
                     newMaterial.setAlphaMode("BLEND");
                     newMaterial.setBaseColorTexture(newTexture);
-                    continue;
+                    break;
                 }
                 if (slot === "specularColorTexture") {
                     logger.info("Pass-through KHR_materials_pbrSpecularGlossiness specularColorTexture");
@@ -368,7 +385,7 @@ async function createTextureLod(args, opts) {
                     const prob = ext.createSpecular();
                     prob.setSpecularColorTexture(newTexture);
                     newMaterial.setExtension(KHRMaterialsSpecular.EXTENSION_NAME, prob);
-                    continue;
+                    break;
                 }
                 else if (slot === "specularTexture") {
                     logger.info("Pass-through KHR_materials_specular specularTexture");
@@ -376,7 +393,7 @@ async function createTextureLod(args, opts) {
                     const prob = ext.createSpecular();
                     prob.setSpecularTexture(newTexture);
                     newMaterial.setExtension(KHRMaterialsSpecular.EXTENSION_NAME, prob);
-                    continue;
+                    break;
                 }
                 logger.warn(`WARN: Unknown texture slot: ${slot} for texture ${textureIndex} → can not load progressively (all slots: ${slots.join(", ")})`);
                 return null;
@@ -524,7 +541,7 @@ async function resizeTexture(options, texture, logger, uri, slots) {
     }
     dstWidth = Math.floor(dstWidth);
     dstHeight = Math.floor(dstHeight);
-    logger.info(`→ Resize texture (${uri || texture.getName() || slots.join(", ")}) from ${srcWidth}x${srcHeight} to ${dstWidth}x${dstHeight}px`);
+    logger.info(`→ Resize texture (${uri || texture.getName() || slots?.join(", ")}) from ${srcWidth}x${srcHeight} to ${dstWidth}x${dstHeight}px`);
     // https://sharp.pixelplumbing.com/api-resize
     const buffer = await sharp(texture.getImage())
         .resize(dstWidth, dstHeight, {

package/dist/utils/test-assert.d.ts

@@ -0,0 +1,8 @@
+export declare class TestAssertionError extends Error {
+    constructor(message: string);
+}
+/**
+ * Lightweight assertion that throws under vitest and is a no-op in production.
+ * Use as a one-liner in pipeline code to catch regressions during tests.
+ */
+export declare function testAssert(condition: boolean, message: string): void;

package/dist/utils/test-assert.js

@@ -0,0 +1,18 @@
+const IS_TEST = process.env.VITEST === 'true';
+export class TestAssertionError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'TestAssertionError';
+    }
+}
+/**
+ * Lightweight assertion that throws under vitest and is a no-op in production.
+ * Use as a one-liner in pipeline code to catch regressions during tests.
+ */
+export function testAssert(condition, message) {
+    if (condition)
+        return;
+    if (IS_TEST) {
+        throw new TestAssertionError(`Assertion failed: ${message}`);
+    }
+}

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.14.0";
+export declare const version = "2.15.0-next.0c5bc39";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.14.0";
+export const version = "2.15.0-next.0c5bc39";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.14.0",
+  "version": "2.15.0-next.0c5bc39",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",
@@ -18,6 +18,12 @@
     "gltf-build-pipeline": "dist/cli/index.js",
     "needle-gltf": "dist/cli/index.js"
   },
+  "files": [
+    "dist",
+    "scripts",
+    "README.md",
+    "CHANGELOG.md"
+  ],
   "imports": {
     "#package.json": "./package.json"
   },
@@ -33,12 +39,18 @@
     "transform:make-progressive": "node scripts/make-progressive.mjs",
     "transform:pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
     "pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
-    "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs"
+    "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs",
+    "test": "tsx tests/utils/download-fixtures.ts && vitest run",
+    "test:watch": "vitest",
+    "test:setup": "tsx tests/utils/download-fixtures.ts"
   },
   "devDependencies": {
-    "@types/node": "^18.7.23",
+    "@types/node": "^22.19.15",
+    "gltf-validator": "^2.0.0-dev.3.9",
     "npm-watch": "^0.13.0",
-    "typescript": "^4.9.5"
+    "tsx": "^4.19.0",
+    "typescript": "^4.9.5",
+    "vitest": "^3.1.1"
   },
   "optionalDependencies": {
     "@img/sharp-darwin-arm64": "^0.33.4",

package/tsconfig.json

@@ -1,20 +0,0 @@
-{
-    "compilerOptions": {
-        "target": "ESNext",
-        "rootDir": "src",
-        "outDir": "dist",
-        "declaration": true,
-        "moduleResolution": "NodeNext",
-        "module": "NodeNext",
-        "esModuleInterop": true,
-        "skipLibCheck": true,
-        "resolveJsonModule": true,
-        "lib": [
-            "ESNext",
-            "DOM"
-        ],
-    },
-    "include": [
-        "src/index.ts",
-    ]
-}