@needle-tools/gltf-build-pipeline 1.1.0 → 1.1.2

package/CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.1.2] - 2023-05-24
+- fix: wasm out of memory error caused by starting to many compression processes at once
+
+## [1.1.1] - 2023-05-18
+- fix: fix progressive texture processing for empty slots array, assume texture slot "baseColorTexture" (for custom shader support)
+
 ## [1.1.0] - 2023-05-06
 - add: support for per glb mesh compression selection (none, draco or meshopt)
 

package/dist/transforms/needle_progressive.js

@@ -23,12 +23,9 @@ function registerExtensions(io) {
 export async function make_progressive(opts) {
     console.log("INFO: Run progressive transform: " + opts.path);
     const file = opts.path;
-    const tasks = [];
-    foreachGLTF(file, f => {
-        const prog = process(f, opts);
-        tasks.push(prog);
+    return foreachGLTF(file, async (f) => {
+        await process(f, opts);
     });
-    await Promise.all(tasks);
 }
 async function process(file, opts) {
     if (!file.endsWith(".glb") && !file.endsWith(".gltf")) {
@@ -50,7 +47,7 @@ async function process(file, opts) {
                 console.log("WARN: \"" + err.message + "\" at \"" + file + "\"");
             }
             else
-                console.log("ERR: \"" + err.message + "\" at \"" + file + "\"");
+                console.log("ERR: \"" + err.message + "\" at \"" + file + "\"\nStacktrace:\n" + err.stack);
             return;
         });
         if (document) {
@@ -76,7 +73,7 @@ async function process(file, opts) {
         }
     }
     catch (err) {
-        console.log("ERR: failed to process file \"" + file + "\", Reason: " + err.message);
+        console.log("ERR: failed to process file \"" + file + "\", Reason: " + err.message, "\nStack:\n" + err.stack);
     }
 }
 // basic idea:
@@ -159,7 +156,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 }
             }
             await resizeTexture(options, texture, logger, uri || name, slots);
-            addToCache(key, texture.getImage());
+            if (opts?.useCache === false) {
+            }
+            else
+                addToCache(key, texture.getImage());
         }
         logger.debug(`Progressive transform finished in ${((Date.now() - startTime) / 1000).toFixed(1)} sec.`);
     });
@@ -181,6 +181,10 @@ function createTextureLod(basePath, level, texture, slots, guid, textureIndex) {
     newTexture.setImage(img);
     newTexture.setURI(texture.getURI());
     newTexture.setMimeType(texture.getMimeType());
+    if (!slots?.length) {
+        console.warn(`WARN: No material slots found for texture at index ${textureIndex} (name: \"${texture.getName()}\") → will assume slot \"baseColorTexture\"`);
+        slots = ["baseColorTexture"];
+    }
     // create material adding the texture to the same slot
     for (const slot of slots) {
         const newMaterial = newDoc.createMaterial();

package/dist/utils/fileutils.d.ts

@@ -1,2 +1,2 @@
 export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
-export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): void;
+export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;

package/dist/utils/fileutils.js

@@ -1,6 +1,6 @@
 import { existsSync, statSync, readdirSync, readFileSync } from 'fs';
 import { isAbsolute } from 'path';
-export function foreachGLTF(args, callback) {
+export async function foreachGLTF(args, callback) {
     // get original directory this run was invoked from
     const initDir = process.env.INIT_CWD;
     const context = {
@@ -10,11 +10,11 @@ export function foreachGLTF(args, callback) {
     // traverse recursively through all files and directories
     if (Array.isArray(args)) {
         for (const arg of args) {
-            testEntry(arg, callback, context);
+            await testEntry(arg, callback, context);
         }
     }
     else if (typeof args === "string") {
-        testEntry(args, callback, context);
+        await testEntry(args, callback, context);
     }
     // try to run with needle config
     if (context.filesFound <= 0 && context.baseDirectory) {
@@ -26,13 +26,13 @@ export function foreachGLTF(args, callback) {
                 // If it's an absolute path
                 if (existsSync(buildDirectory)) {
                     console.log("INFO: Using build directory from needle.config.json: " + buildDirectory);
-                    testEntry(buildDirectory, callback, context);
+                    await testEntry(buildDirectory, callback, context);
                 }
                 else { // if its a relative path
                     const newPath = context.baseDirectory + "/" + buildDirectory;
                     if (existsSync(newPath)) {
                         console.log("INFO: Using build directory from needle.config.json: " + newPath);
-                        testEntry(newPath, callback, context);
+                        await testEntry(newPath, callback, context);
                     }
                 }
             }
@@ -42,7 +42,7 @@ export function foreachGLTF(args, callback) {
         console.log("WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory. This path may be relative to " + initDir);
     }
 }
-function testEntry(entry, callback, context) {
+async function testEntry(entry, callback, context) {
     if (entry === "." || entry === "./") {
         if (context.baseDirectory) {
             entry = context.baseDirectory;
@@ -82,7 +82,10 @@ function testEntry(entry, callback, context) {
     }
     if (entry.endsWith(".glb") || entry.endsWith(".gltf")) {
         context.filesFound += 1;
-        callback(entry);
+        const res = callback(entry);
+        if (res instanceof Promise) {
+            await res;
+        }
     }
     else {
         const stat = statSync(entry);
@@ -93,7 +96,7 @@ function testEntry(entry, callback, context) {
             }
             const files = readdirSync(entry);
             for (const file of files) {
-                testEntry(entry + "/" + file, callback, context);
+                await testEntry(entry + "/" + file, callback, context);
             }
         }
         else {

package/pack-gltf.mjs

@@ -69,6 +69,9 @@ if (noToktx)
     console.log("Using toktx is disabled");
 
 const packedFiles = new Set();
+let totalGPUMemoryInMB = 0;
+let totalFilesProcessed = 0;
+let totalFileSizeInMB = 0;
 
 async function packFile(file) {
     if (!existsSync(file)) {
@@ -105,8 +108,8 @@ async function packFile(file) {
         await document.transform(
             needle_asset(),
             needle_texture_transform({
-                useCache: allowCaching, 
-                noToktx: noToktx, 
+                useCache: allowCaching,
+                noToktx: noToktx,
                 debug: debug,
                 reportGPUMemory: val => totalGPUTextureMemmory += val
             }),
@@ -117,10 +120,14 @@ async function packFile(file) {
         if (debug) file += ".debug.glb";
         await io.write(file, document);
         const duration = Date.now() - startTime;
-        const sizeInBytes = statSync(file).size;
+        const sizeInMB = statSync(file).size / 1024 / 1024;
         const filename = path.basename(file);
         const gpuMemoryInMB = totalGPUTextureMemmory / 1024 / 1024;
-        console.log(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${(prevSize / 1024 / 1024).toFixed(1)} MB → ${(sizeInBytes / 1024 / 1024).toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${file}`);
+        console.log(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${(prevSize / 1024 / 1024).toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${file}`);
+
+        totalFilesProcessed++;
+        totalFileSizeInMB += sizeInMB;
+        totalGPUMemoryInMB += gpuMemoryInMB;
         return true;
     }
     catch (err) {
@@ -130,7 +137,11 @@ async function packFile(file) {
     }
 }
 
-
+let startTime = Date.now();
 foreachGLTF(process.argv, async (file) => {
     await packFile(file);
-});
+}).then(() => {
+    const duration = Date.now() - startTime;
+    const durationInMinutes = duration / 1000 / 60;
+    console.log(`Finished packing ${totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size: ${totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${totalGPUMemoryInMB.toFixed(1)} MB`);
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",