@needle-tools/gltf-build-pipeline 2.15.8 → 2.15.9-next.aba648c

package/dist/extensions/NEEDLE_lightmaps/index.js

@@ -3,6 +3,8 @@ import { existsSync, readFileSync } from 'fs';
 import { join, dirname } from 'path';
 import { isLightmap } from '../../utils/texture.js';
 import { getExtensionInputFile } from '../index.js';
+import { findSource, getTexture } from '../utils.js';
+import { testAssert } from '../../utils/test-assert.js';
 const EXTENSION_NAME = 'NEEDLE_lightmaps';
 /**
  * Opaque-like ExtensionProperty that holds NEEDLE_lightmaps data on the Root.
@@ -66,7 +68,8 @@ export class NEEDLE_lightmaps_ext extends Extension {
                 continue;
             const exrPath = inputDir ? join(inputDir, entry.pointer) : '';
             if (!exrPath || !existsSync(exrPath)) {
-                this.document.getLogger().warn(`NEEDLE_lightmaps: External EXR not found: ${exrPath || entry.pointer}`);
+                this.document.getLogger().error(`NEEDLE_lightmaps: External EXR not found: ${exrPath || entry.pointer}`);
+                testAssert(false, `NEEDLE_lightmaps: External EXR not found: ${exrPath || entry.pointer}`);
                 continue;
             }
             // Read EXR bytes and inject into JSON so the core reader creates a Texture
@@ -113,12 +116,7 @@ export class NEEDLE_lightmaps_ext extends Extension {
                 const texDef = textureDefs[texIndex];
                 if (texDef) {
                     inputTextureDef = texDef;
-                    const imageIndex = texDef.source
-                        ?? texDef.extensions?.['EXT_texture_exr']?.source
-                        ?? texDef.extensions?.['KHR_texture_basisu']?.source;
-                    if (imageIndex !== undefined) {
-                        texture = context.textures[imageIndex] ?? null;
-                    }
+                    texture = getTexture(context, texDef) ?? null;
                 }
             }
             else if (entry.pointer.match(/\.exr$/i)) {
@@ -126,7 +124,19 @@ export class NEEDLE_lightmaps_ext extends Extension {
                 const imageIndex = this.injectedImageIndices.get(entry.pointer);
                 if (imageIndex !== undefined) {
                     texture = context.textures[imageIndex] ?? null;
-                    inputTextureDef = textureDefs.find(t => t.source === imageIndex);
+                    inputTextureDef = textureDefs.find(t => findSource(t) === imageIndex);
+                }
+            }
+            else if (!entry.pointer.startsWith('/')) {
+                // Already-processed external pointer (e.g. "sunrise.pmrem.ktx2")
+                // Find the texture by matching the URI in the existing images
+                const images = json.images ?? [];
+                for (let i = 0; i < images.length; i++) {
+                    if (images[i].uri === entry.pointer) {
+                        texture = context.textures[i] ?? null;
+                        inputTextureDef = textureDefs.find(t => findSource(t) === i);
+                        break;
+                    }
                 }
             }
             if (texture) {
@@ -141,7 +151,8 @@ export class NEEDLE_lightmaps_ext extends Extension {
                 }
             }
             else {
-                logger.warn(`NEEDLE_lightmaps: Could not resolve texture for pointer "${entry.pointer}"`);
+                logger.error(`NEEDLE_lightmaps: Could not resolve texture for pointer "${entry.pointer}"`);
+                testAssert(false, `NEEDLE_lightmaps: Could not resolve texture for pointer "${entry.pointer}"`);
             }
         }
         // Lightmap workaround: prevent compression on lightmap textures (non-EXR)
@@ -217,11 +228,7 @@ export class NEEDLE_lightmaps_ext extends Extension {
         const textures = context.jsonDoc.json.textures = context.jsonDoc.json.textures || [];
         // Try to find an existing texture def pointing to the same image
         for (let i = 0; i < textures.length; i++) {
-            const texDef = textures[i];
-            const source = texDef.source
-                ?? texDef.extensions?.['KHR_texture_basisu']?.source
-                ?? texDef.extensions?.['NEEDLE_pmrem']?.source;
-            if (source === imageIndex) {
+            if (findSource(textures[i]) === imageIndex) {
                 return i;
             }
         }

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -1,5 +1,5 @@
 import { Extension, ExtensionProperty, PropertyType, Node, TextureChannel } from '@gltf-transform/core';
-import { getTexture, tryGetInArray, writeExtension } from '../utils.js';
+import { findSource, getTexture, tryGetInArray, writeExtension } from '../utils.js';
 import { isLightmap } from '../../utils/texture.js';
 import { NEEDLE_progressive_texture_settings } from '../NEEDLE_progressive_texture_settings/index.js';
 import { testAssert } from '../../utils/test-assert.js';
@@ -655,7 +655,8 @@ class JsonPointerHandler {
                             this.externalExrPointers.push({ obj, key, texture });
                         }
                         else if (exrPath) {
-                            console.warn(`WARN: External EXR not found: ${exrPath}`);
+                            this.document.getLogger().error(`NEEDLE_opaque: External EXR not found: ${exrPath}`);
+                            testAssert(false, `NEEDLE_opaque: External EXR not found: ${exrPath}`);
                         }
                     }
                     if (value.startsWith("/materials/")) {
@@ -999,22 +1000,6 @@ class TextureIndexResolver extends PointerResolver {
         return json.samplers.length - 1;
     }
 }
-function findSource(texDef) {
-    if (texDef.source !== undefined) {
-        return texDef.source;
-    }
-    if (texDef.extensions) {
-        const keys = Object.keys(texDef.extensions);
-        for (let i = 0; i < keys.length; i++) {
-            const key = keys[i];
-            const ext = texDef.extensions[key];
-            if (ext["source"] !== undefined) {
-                return ext["source"];
-            }
-        }
-    }
-    return undefined;
-}
 class MaterialIndexResolver extends PointerResolver {
     init(context) {
     }

package/dist/extensions/utils.d.ts

@@ -3,4 +3,7 @@ import { GLTF } from '@gltf-transform/core';
 export declare function tryGetInArray<T>(arr: Array<T> | undefined | null, index: number | undefined): T | null;
 export declare function writeExtension(prop: GLTF.IProperty | null, name: string, ext: any): void;
 export declare function getTexture(context: ReaderContext, tex: GLTF.ITexture): Texture;
+/** Resolve the image source index from a texture definition, checking `texDef.source`
+ *  and then iterating all extension keys for a `.source` property. */
+export declare function findSource(texDef: GLTF.ITexture): number | undefined;
 export declare function getTextureDefinition(context: WriterContext | ReaderContext, tex: Texture): GLTF.ITexture | null;

package/dist/extensions/utils.js

@@ -21,19 +21,26 @@ export function writeExtension(prop, name, ext) {
 export function getTexture(context, tex) {
     if (!context.jsonDoc.json.images)
         return null;
-    if (tex.source !== undefined) {
-        return context.textures[tex.source];
+    const source = findSource(tex);
+    if (source !== undefined) {
+        return context.textures[source];
     }
-    else {
-        for (const key of Object.keys(tex.extensions)) {
-            const ext = tex.extensions[key];
-            if (ext && typeof ext["source"] === "number") {
-                return context.textures[ext["source"]];
-            }
+    console.log("Could not find texture", tex);
+    return null;
+}
+/** Resolve the image source index from a texture definition, checking `texDef.source`
+ *  and then iterating all extension keys for a `.source` property. */
+export function findSource(texDef) {
+    if (texDef.source !== undefined)
+        return texDef.source;
+    if (texDef.extensions) {
+        for (const key of Object.keys(texDef.extensions)) {
+            const ext = texDef.extensions[key];
+            if (ext && ext["source"] !== undefined)
+                return ext["source"];
         }
     }
-    console.log("Could not fnd texture", tex);
-    return null;
+    return undefined;
 }
 export function getTextureDefinition(context, tex) {
     const textureDefinitions = context.jsonDoc.json.textures;

package/dist/scripts/pack-gltf.js

@@ -7,7 +7,7 @@ import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 import { dedup, metalRough, prune, resample } from '@gltf-transform/functions';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, setExtensionInputFile } from '../extensions/index.js';
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
-import { getOutputPath, getVersion, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
+import { copyExternalResources, getOutputPath, getVersion, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
 import { TestAssertionError } from '../utils/test-assert.js';
 /** Compress a glTF file
@@ -78,6 +78,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             /** @ts-ignore */
             document.getLogger().verbosity = options.verbose ? Verbosity.DEBUG : options.debug ? Verbosity.INFO : Verbosity.WARN;
         }
+        copyExternalResources(document, assetSourceFile, outputFile);
         let totalGPUTextureMemory = 0;
         /** @type {Array<import('@gltf-transform/core').Transform>} */
         const transforms = [

package/dist/transforms/needle_pmrem.js

@@ -8,6 +8,7 @@ import { v4 as uuid } from 'uuid';
 import { createHash } from 'crypto';
 import tmp from 'tmp';
 import { addToCache, tryGetFromCache } from '../cache/index.js';
+import { testAssert } from '../utils/test-assert.js';
 /** Resolve the tools/pmrem/ directory relative to this file.
  *  At runtime this file is at dist/transforms/needle_pmrem.js,
  *  so tools/pmrem/ is at ../../tools/pmrem/ */
@@ -61,12 +62,14 @@ export class NeedlePmremContext {
         const wasmBinPath = join(this.pmremDir, 'pkg', 'pmrem_wasm_bg.wasm');
         if (!existsSync(wasmJsPath) || !existsSync(wasmBinPath)) {
             this.logger.warn('pmrem: WASM not found — EXR textures will not be processed');
+            testAssert(false, 'pmrem WASM files are missing. Please run `npm run build:pmrem` to build the tools/pmrem/ WASM module.');
             return false;
         }
         // Check basisu binary
         this.basisuExe = resolveBasisuExe(this.pmremDir);
         if (!existsSync(this.basisuExe)) {
             this.logger.warn(`pmrem: basisu binary not found at ${this.basisuExe} — EXR textures will not be processed`);
+            testAssert(false, `basisu executable is missing for platform ${process.platform}. Please run \`npm run build:pmrem\` to build the tools/pmrem/ binaries.`);
             return false;
         }
         // Set executable permission
@@ -85,6 +88,7 @@ export class NeedlePmremContext {
         }
         catch (err) {
             this.logger.warn(`pmrem: Failed to load WASM — ${err.message}`);
+            testAssert(false, `Failed to load pmrem WASM module: ${err.message}`);
             return false;
         }
         // Create temp directory (per-instance to avoid parallel cleanup races)
@@ -109,6 +113,7 @@ export class NeedlePmremContext {
         const contentHash = createHash('md5').update(exrBytes).digest('hex');
         const cacheKey = `pmrem-${contentHash}`;
         // Get KTX2 bytes — deduplicated and optionally cached
+        const wasDeduplicated = this.processingPromises.has(cacheKey);
         const ktx2Bytes = await this.getOrProcessExr(cacheKey, exrBytes, index, name, context.useCache);
         // Remove EXT_texture_exr property since the texture is now KTX2
         const exrProp = texture.getExtension('EXT_texture_exr');
@@ -133,20 +138,25 @@ export class NeedlePmremContext {
             texture.setName(ktx2FileName);
             // Clear image data so gltf-transform doesn't embed it in the GLB
             texture.setImage(null);
-            this.logger.info(`pmrem:texture[${index}] ${name}: external → ${ktx2FileName}`);
+            this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: external → ${ktx2FileName}`);
         }
         else {
             // Bundled: embed the KTX2 in the GLB
             texture.setImage(new Uint8Array(ktx2Bytes));
         }
-        // Report size
-        const sizeStr = image.byteLength < 100 * 1024
-            ? `${(image.byteLength / 1024).toFixed(0)} KB`
-            : `${(image.byteLength / (1024 * 1024)).toFixed(2)} MB`;
-        const dstSizestr = ktx2Bytes.length < 100 * 1024
-            ? `${(ktx2Bytes.length / 1024).toFixed(0)} KB`
-            : `${(ktx2Bytes.length / (1024 * 1024)).toFixed(2)} MB`;
-        this.logger.info(`pmrem:texture[${index}] ${name}: ${sizeStr} EXR → ${dstSizestr} KTX2`);
+        // Report size — skip info log for deduplicated textures to avoid confusion
+        if (wasDeduplicated) {
+            this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: reused (identical content) → ${texture.getURI() || texture.getName() || name}`);
+        }
+        else {
+            const sizeStr = image.byteLength < 100 * 1024
+                ? `${(image.byteLength / 1024).toFixed(0)} KB`
+                : `${(image.byteLength / (1024 * 1024)).toFixed(2)} MB`;
+            const dstSizestr = ktx2Bytes.length < 100 * 1024
+                ? `${(ktx2Bytes.length / 1024).toFixed(0)} KB`
+                : `${(ktx2Bytes.length / (1024 * 1024)).toFixed(2)} MB`;
+            this.logger.info(`NEEDLE_pmrem[${index}] ${name}: ${sizeStr} EXR → ${dstSizestr} KTX2 (${texture.getURI() || texture.getName() || name})`);
+        }
         this.convertedTextures.push(texture);
         this.processed++;
         return true;
@@ -156,7 +166,7 @@ export class NeedlePmremContext {
         // In-memory dedup: if we're already processing identical EXR content, reuse the promise
         const existing = this.processingPromises.get(cacheKey);
         if (existing) {
-            this.logger.info(`pmrem:texture[${index}] ${name}: reusing in-flight result (identical content)`);
+            this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: reusing in-flight result (identical content)`);
             return existing;
         }
         const promise = this.processExrToKtx2(cacheKey, exrBytes, index, name, useCache);
@@ -168,24 +178,32 @@ export class NeedlePmremContext {
         if (useCache) {
             const cached = tryGetFromCache(cacheKey);
             if (cached && cached.length > 0) {
-                this.logger.info(`pmrem:texture[${index}] ${name}: loaded from cache`);
+                this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: loaded from cache`);
                 return cached;
             }
         }
-        // 1. Run PMREM on the EXR bytes
-        this.logger.info(`pmrem:texture[${index}] ${name}: Running PMREM...`);
+        // 1. Run PMREM on the EXR bytes (synchronous WASM — may take a while)
+        const inputSize = exrBytes.byteLength < 100 * 1024
+            ? `${(exrBytes.byteLength / 1024).toFixed(0)} KB`
+            : `${(exrBytes.byteLength / (1024 * 1024)).toFixed(2)} MB`;
+        this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: Running PMREM WASM (${inputSize} EXR) — this may take a while...`);
+        // Yield to allow log to flush before blocking WASM call
+        await new Promise(r => setTimeout(r, 0));
+        const t0 = Date.now();
         const pmremExrBytes = this.wasmModule.pmrem_exr(exrBytes);
+        const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+        this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: PMREM done in ${elapsed}s (${(pmremExrBytes.byteLength / 1024 / 1024).toFixed(1)} MB)`);
         // 2. Write PMREM EXR to temp file
         const id = uuid();
         const tempExr = join(this.tmpDir, `${id}.pmrem.exr`);
         const tempKtx2 = join(this.tmpDir, `${id}.ktx2`);
         writeFileSync(tempExr, Buffer.from(pmremExrBytes));
         // 3. Encode to KTX2 HDR via basisu
-        this.logger.debug(`pmrem:texture[${index}] ${name}: Encoding KTX2 HDR...`);
+        this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: Encoding KTX2 HDR...`);
         const variant = 'hdr4x4'; // 6x6 needs an update in Needle
         await encodeKTX2WithBasisu(this.basisuExe, tempExr, tempKtx2, variant, true);
         if (!existsSync(tempKtx2)) {
-            throw new Error(`pmrem:texture[${index}] ${name}: basisu did not produce output`);
+            throw new Error(`NEEDLE_pmrem[${index}] ${name}: basisu did not produce output`);
         }
         // 4. Read KTX2 result
         const ktx2Bytes = readFileSync(tempKtx2);

package/dist/transforms/needle_progressive.js

@@ -8,7 +8,7 @@ import path, { basename, resolve } from 'path';
 import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
 import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
-import { detectTextureCompressionMode, ensureIsDirectory, foreachGLTF, generateGuid, getOutputPath, isLightmap, setLightmapName, ioTryReadWithMissingResources, validateImageMimeType, writeNodeIO } from '../utils/index.js';
+import { copyExternalResources, detectTextureCompressionMode, ensureIsDirectory, foreachGLTF, generateGuid, getOutputPath, isLightmap, setLightmapName, ioTryReadWithMissingResources, validateImageMimeType, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptEncoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive, determineMeshCompression } from './needle_mesh_transform.js';
@@ -88,6 +88,9 @@ async function onProcess(file, opts) {
                 document.getLogger().verbosity = (opts.debug || opts.verbose) ? Verbosity.DEBUG : Verbosity.WARN;
             }
             const logger = opts.logger || document.getLogger();
+            if (opts.outpath) {
+                copyExternalResources(document, file, path.join(opts.outpath, path.basename(file)));
+            }
             let smallesTextureSize = 128;
             if (opts.config.usecase === "product") {
                 smallesTextureSize = 512;

package/dist/utils/nodeio.d.ts

@@ -4,3 +4,8 @@ import { NodeIO, Document, PlatformIO } from "@gltf-transform/core";
  */
 export declare function writeNodeIO(nodeio: NodeIO, filename: string, document: Document): Promise<string>;
 export declare function ioTryReadWithMissingResources(io: PlatformIO, filepath: string, iteration?: number): Promise<Document>;
+/**
+ * Copy external resources (images/buffers with URIs) referenced by a Document
+ * from the input directory to the output directory. No-op if input and output resolve to the same directory.
+ */
+export declare function copyExternalResources(document: Document, inputFile: string, outputFile: string): void;

package/dist/utils/nodeio.js

@@ -1,4 +1,4 @@
-import { existsSync, mkdirSync } from "fs";
+import { cpSync, existsSync, mkdirSync } from "fs";
 import { writeFile } from "fs/promises";
 import path from "path";
 import sharp from "sharp";
@@ -84,3 +84,33 @@ export async function ioTryReadWithMissingResources(io, filepath, iteration = 0)
     }
     return document;
 }
+/**
+ * Copy external resources (images/buffers with URIs) referenced by a Document
+ * from the input directory to the output directory. No-op if input and output resolve to the same directory.
+ */
+export function copyExternalResources(document, inputFile, outputFile) {
+    const inputDir = path.resolve(path.dirname(inputFile));
+    const outputDir = path.resolve(path.dirname(outputFile));
+    if (inputDir === outputDir)
+        return;
+    for (const texture of document.getRoot().listTextures()) {
+        const uri = texture.getURI();
+        if (!uri || uri.startsWith('data:'))
+            continue;
+        const src = path.join(inputDir, uri);
+        const dst = path.join(outputDir, uri);
+        if (!existsSync(dst) && existsSync(src)) {
+            cpSync(src, dst);
+        }
+    }
+    for (const buffer of document.getRoot().listBuffers()) {
+        const uri = buffer.getURI();
+        if (!uri || uri.startsWith('data:'))
+            continue;
+        const src = path.join(inputDir, uri);
+        const dst = path.join(outputDir, uri);
+        if (!existsSync(dst) && existsSync(src)) {
+            cpSync(src, dst);
+        }
+    }
+}

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.15.8";
+export declare const version = "2.15.9-next.aba648c";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.15.8";
+export const version = "2.15.9-next.aba648c";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.15.8",
+  "version": "2.15.9-next.aba648c",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",