@needle-tools/gltf-build-pipeline 1.0.0 → 1.0.2

package/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.0.2] - 2023-04-19
+- change: progressive should not log error out when running for already compressed gltf
+- change: ignore EXT_texture_exr extension re-assignment
+
+## [1.0.1] - 2023-04-17
+- fix: register meshopt decoder 
+- fix: catch exception in document read
+- change: log gpu memory estimation for textures
+
 ## [1.0.0] - 2023-04-04
 - change: file utils use needle.config.json to find files if no directory was specified
 - change: add generator info

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -400,7 +400,8 @@ class TextureHandler {
             }
         });
         if (foundTexture === false) {
-            console.warn("WARN: failed to re-assign extension:", this.key, this.index, this.tryGetSourceIndex(this.textureDefinition), textureDefs.length);
+            if (this.key !== "EXT_texture_exr" || this.debug)
+                console.warn("WARN: failed to re-assign extension:", this.key, this.index, this.tryGetSourceIndex(this.textureDefinition), textureDefs.length);
         }
         // this.document
         //     .getRoot()

package/dist/transforms/needle_common.d.ts

@@ -1,6 +1,7 @@
 import { Document, Texture } from '@gltf-transform/core';
 import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
 export interface NeedleTransformContext {
+    debug?: boolean;
     useCache: boolean;
     noToktx: boolean;
 }

package/dist/transforms/needle_progressive.js

@@ -8,6 +8,7 @@ import { addToCache, getKey, tryGetFromCache } from '../cache/index.js';
 import sharp from "sharp";
 import { foreachGLTF } from '../utils/index.js';
 import { needle_asset } from './needle_asset.js';
+import { MeshoptDecoder } from 'meshoptimizer';
 function registerExtensions(io) {
     registerDefaultExtensions(io);
     io.registerExtensions([
@@ -37,32 +38,45 @@ async function process(file, opts) {
     if (!existsSync(file)) {
         console.log("ERR: file does not exist: " + file);
     }
-    const io = new NodeIO();
-    registerExtensions(io);
-    const document = await io.read(file).catch((err) => {
-        console.log("ERR: failed to read file \"" + file + "\", Reason: " + err.message);
-        return;
-    });
-    if (document) {
-        if (opts.debug === true) {
-            if (file.includes(".debug.")) {
-                console.log("Skip debug file: " + file);
-                return;
+    try {
+        const io = new NodeIO();
+        registerExtensions(io);
+        io.registerDependencies({
+            'meshopt.decoder': MeshoptDecoder,
+        });
+        const document = await io.read(file).catch((err) => {
+            // Log this message as a warning, it happens when trying to run on a already compressed gltf file (e.g. pre-processed. When logging an error we fail the build which is not desired)
+            if (err.message === "Cannot read properties of undefined (reading 'DT_FLOAT32')") {
+                console.log("WARN: \"" + err.message + "\" at \"" + file + "\"");
             }
+            else
+                console.log("ERR: \"" + err.message + "\" at \"" + file + "\"");
+            return;
+        });
+        if (document) {
+            if (opts.debug === true) {
+                if (file.includes(".debug.")) {
+                    console.log("Skip debug file: " + file);
+                    return;
+                }
+            }
+            //@ts-ignore
+            document.getLogger().verbosity = 0;
+            await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts));
+            let filename = file;
+            if (opts.debug === true) {
+                // get extension
+                const extension = filename.split('.').pop();
+                // remove extension
+                filename = filename.split('.').slice(0, -1).join('.');
+                // add new extension
+                filename = filename + ".debug." + extension;
+            }
+            io.write(filename, document);
         }
-        //@ts-ignore
-        document.getLogger().verbosity = 0;
-        await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts));
-        let filename = file;
-        if (opts.debug === true) {
-            // get extension
-            const extension = filename.split('.').pop();
-            // remove extension
-            filename = filename.split('.').slice(0, -1).join('.');
-            // add new extension
-            filename = filename + ".debug." + extension;
-        }
-        io.write(filename, document);
+    }
+    catch (err) {
+        console.log("ERR: failed to process file \"" + file + "\", Reason: " + err.message);
     }
 }
 // basic idea:

package/dist/transforms/needle_texture_transform.js

@@ -1,3 +1,4 @@
+import { ImageUtils } from "@gltf-transform/core";
 import { resizeTextureToMax } from "./needle_progressive.js";
 import { NEEDLE_compression_texture } from '../extensions/index.js';
 import { addToCache, getKey, tryGetFromCache } from "../cache/index.js";
@@ -25,7 +26,8 @@ export async function resizeTextureIfNecessary(texture) {
 export const needle_texture_transform = function (options) {
     const webPContext = new NeedleWebPContext({});
     const toktxContext = new NeedleToktxContext({
-        jobs: undefined, // default to number of cores
+        jobs: undefined,
+        debug: options.debug,
     });
     const context = {
         ...options,
@@ -39,9 +41,30 @@ export const needle_texture_transform = function (options) {
         try {
             const logger = document.getLogger();
             const textures = document.getRoot().listTextures();
+            let totalGpuSize = 0;
             await Promise.all(textures.map((texture, textureIndex) => {
-                return processTexture(texture, textureIndex, document, logger, context);
+                return processTexture(texture, textureIndex, document, logger, context).then(() => {
+                    try {
+                        const size = ImageUtils.getVRAMByteLength(texture.getImage(), texture.getMimeType());
+                        totalGpuSize += size;
+                    }
+                    catch {
+                        // Ignore
+                    }
+                });
             }));
+            if (totalGpuSize > 0) {
+                // TODO: report this to the parent
+                const gpuMemoryEstimate = totalGpuSize / 1024 / 1024;
+                if (context.reportGPUMemory)
+                    context.reportGPUMemory(totalGpuSize);
+                else {
+                    console.log(`> Total GPU size: ${(gpuMemoryEstimate).toFixed(2)} MB of ${textures.length} textures`);
+                }
+                if (gpuMemoryEstimate > 1000) {
+                    console.log(`WARN: Total estimated GPU size is very large - this may cause issues on some devices. Consider changing the compression settings or reducing texture size.`);
+                }
+            }
         }
         finally {
             await webPContext.finalize(document);

package/dist/transforms/needle_toktx.d.ts

@@ -3,6 +3,7 @@ import { KHRTextureBasisu } from "@gltf-transform/extensions";
 import { NEEDLE_compression_texture_schema } from "../extensions/index.js";
 import { INeedleTextureTransformContext } from "./needle_common.js";
 export interface NeedleToktxOptions {
+    debug?: boolean;
     jobs?: number;
     toktxPath?: string;
 }

package/dist/transforms/needle_toktx.js

@@ -9,6 +9,7 @@ import os from 'os';
 import tmp from 'tmp';
 import pLimit from 'p-limit';
 const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
+let didLogEnvironmentInfo = false;
 export class NeedleToktxContext {
     limit;
     basisuExtension;
@@ -31,15 +32,19 @@ export class NeedleToktxContext {
         // Use global env variable
         if (process.env.NEEDLE_TOKTX?.length) {
             if (!existsSync(process.env.NEEDLE_TOKTX)) {
-                console.log("WARN: NEEDLE_TOKTX is set to a non-existing path: \"" + process.env.NEEDLE_TOKTX + "\"");
+                if (options.debug)
+                    console.log("WARN: NEEDLE_TOKTX is set to a non-existing path: \"" + process.env.NEEDLE_TOKTX + "\"");
             }
             else {
-                console.log("INFO: Adding NEEDLE_TOKTX to PATH: \"" + process.env.NEEDLE_TOKTX + "\"");
+                if (options.debug)
+                    console.log("INFO: Adding NEEDLE_TOKTX to PATH: \"" + process.env.NEEDLE_TOKTX + "\"");
                 this.env.PATH = `${process.env.NEEDLE_TOKTX};${this.env.PATH}`;
             }
         }
         else {
-            console.log("INFO: Environment variable 'NEEDLE_TOKTX' not set. Use 'NEEDLE_TOKTX' to specify the directory of the toktx installation (path/to/bin)");
+            if (didLogEnvironmentInfo === false)
+                console.log("INFO: Environment variable 'NEEDLE_TOKTX' not set. Use 'NEEDLE_TOKTX' to specify the directory of the toktx installation (path/to/bin)");
+            didLogEnvironmentInfo = true;
         }
         // fallback default installation directory:
         if (isUsingWindows) {
@@ -50,7 +55,7 @@ export class NeedleToktxContext {
         // append default system paths
         if (isUsingWindows && process.env.Path?.length)
             this.env.PATH = `${this.env.PATH};${process.env.Path}`;
-        if (process.argv?.includes("debug")) {
+        if (options.debug) {
             const ktxPaths = this.env.PATH.split(";").filter(p => p.toLocaleLowerCase().includes("ktx"));
             console.log("INFO: Found these paths including ktx:\n\t" + ktxPaths.join("\n\t") + "");
         }

package/dist/transforms/toktx.js

@@ -167,6 +167,7 @@ export function createParams(slots, channels, size, logger, numTextures, options
     }
     return params;
 }
+let didLogToktxVersion = false;
 export async function checkKTXSoftware(logger, env) {
     const [status, stdout, stderr] = await waitExit(spawn('toktx', ['--version'], { env: env }));
     const version = (stdout || stderr)
@@ -180,7 +181,9 @@ export async function checkKTXSoftware(logger, env) {
         logger.warn(`toktx: Expected KTX-Software >= v${KTX_SOFTWARE_VERSION_MIN}, found ${version}.`);
     }
     else {
-        logger.debug(`toktx: Found KTX-Software ${version}.`);
+        if (!didLogToktxVersion)
+            logger.debug(`toktx: Found KTX-Software ${version}.`);
+        didLogToktxVersion = true;
     }
     return semver.clean(version);
 }

package/dist/utils/fileutils.js

@@ -51,12 +51,13 @@ function testEntry(entry, callback, context) {
     if (existsSync(entry) === false) {
         // If no base directory is defined we cannot try to resolve the path
         if (context.baseDirectory === undefined) {
+            console.log("WARN: Path does not exist: " + entry);
             return;
         }
         // Try resolve potential relative path
         const newPath = context.baseDirectory + "/" + entry;
-        console.log(newPath);
         if (existsSync(newPath) === false) {
+            console.log(`INFO: Resolved path failed: ${entry} -> ${newPath}`);
             return;
         }
         else {
@@ -86,10 +87,17 @@ function testEntry(entry, callback, context) {
     else {
         const stat = statSync(entry);
         if (stat.isDirectory()) {
+            // Don't traverse node_modules
+            if (entry.endsWith("node_modules")) {
+                return;
+            }
             const files = readdirSync(entry);
             for (const file of files) {
                 testEntry(entry + "/" + file, callback, context);
             }
         }
+        else {
+            // console.log("WARN: Ignoring file: " + entry);
+        }
     }
 }

package/pack-gltf.mjs

@@ -13,6 +13,7 @@ import { needle_texture_transform } from "./dist/transforms/needle_texture_trans
 import { needle_animation_transform } from "./dist/transforms/needle_animation_transform.js";
 import { foreachGLTF } from './fileutils.mjs';
 import { needle_asset } from './dist/transforms/needle_asset.js';
+import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 
 const args = process.argv;
 const showHelp = args.includes("help");
@@ -82,35 +83,51 @@ async function packFile(file) {
     packedFiles.add(file);
     console.log("→ Begin packing: " + file);
 
-    const prevSize = statSync(file).size;
-    const startTime = Date.now();
-    const io = new NodeIO()
-        .registerExtensions(ALL_EXTENSIONS)
-        .registerExtensions(NEEDLE_EXTENSIONS)
-        .registerExtensions([
-            NEEDLE_compression_texture,
-            NEEDLE_mesh_compression,
-        ])
-        .registerDependencies({
-            'draco3d.decoder': await draco3d.createDecoderModule(),
-            'draco3d.encoder': await draco3d.createEncoderModule(),
-        });
-    const document = await io.read(file);
-    document.getLogger().verbosity = 0;
-    await document.transform(
-        needle_asset(),
-        needle_texture_transform({ useCache: allowCaching, noToktx: noToktx }),
-        needle_mesh_transform({ ratio: 0.75, error: 0.0005 }),
-        needle_animation_transform(),
-        draco(),
-    );
-    if (debug) file += ".debug.glb";
-    await io.write(file, document);
-    const duration = Date.now() - startTime;
-    const sizeInBytes = statSync(file).size;
-    const filename = path.basename(file);
-
-    console.log(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${(prevSize / 1024 / 1024).toFixed(1)} MB → ${(sizeInBytes / 1024 / 1024).toFixed(1)} MB: ${filename} at ${file}`);
+    try {
+        const prevSize = statSync(file).size;
+        const startTime = Date.now();
+        const io = new NodeIO()
+            .registerExtensions(ALL_EXTENSIONS)
+            .registerExtensions(NEEDLE_EXTENSIONS)
+            .registerExtensions([
+                NEEDLE_compression_texture,
+                NEEDLE_mesh_compression,
+            ])
+            .registerDependencies({
+                'draco3d.decoder': await draco3d.createDecoderModule(),
+                'draco3d.encoder': await draco3d.createEncoderModule(),
+                'meshopt.decoder': MeshoptDecoder,
+                'meshopt.encoder': MeshoptEncoder,
+            });
+        const document = await io.read(file);
+        document.getLogger().verbosity = 0;
+        let totalGPUTextureMemmory = 0;
+        await document.transform(
+            needle_asset(),
+            needle_texture_transform({
+                useCache: allowCaching, 
+                noToktx: noToktx, 
+                debug: debug,
+                reportGPUMemory: val => totalGPUTextureMemmory += val
+            }),
+            needle_mesh_transform({ ratio: 0.75, error: 0.0005 }),
+            needle_animation_transform(),
+            draco(),
+        );
+        if (debug) file += ".debug.glb";
+        await io.write(file, document);
+        const duration = Date.now() - startTime;
+        const sizeInBytes = statSync(file).size;
+        const filename = path.basename(file);
+        const gpuMemoryInMB = totalGPUTextureMemmory / 1024 / 1024;
+        console.log(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${(prevSize / 1024 / 1024).toFixed(1)} MB → ${(sizeInBytes / 1024 / 1024).toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${file}`);
+        return true;
+    }
+    catch (err) {
+        console.error("ERR: packing " + file + " failed: " + err.message);
+        if (debug) throw err;
+        return false;
+    }
 }
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",