@needle-tools/gltf-build-pipeline 2.0.0-alpha → 2.1.0-alpha.1

package/CHANGELOG.md

@@ -4,6 +4,36 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.1.0-alpha] - 2023-06-03
+- add: compression config support for high-level usecases that can be passed via commandline arguments or via config. Available options are currently `default`, `world` and `product`.  
+- add: texture LODs can now be generated by passing in the path to a compression config. `--config <path/to/config.json>` 
+- change: VRMs now use draco mesh compression by default 
+
+### Config Examples
+  - `default` - use default compression. When not explictly configured textures will be compressed with ETC1S
+  - `product` - when not explictly configured textures will use UASTC for `metallicRoughnessTexture` and `normalTexture`   
+  - `world` - generate texture LODs when enabled (and not explictly disabled by `textures: { lods: false }`)
+  ```json
+  {
+    "usecase": "default" // optional
+    "textures" : { 
+        "lods": true
+    }
+  }
+  ```
+  The config can also be part of a `needle.config.json`:
+  ```json
+  {
+    ... regular needle.config
+    "gltf": {
+        "usecase": "default" // optional
+        "textures": {
+            "lods": true
+        }
+    }
+  }
+  ```
+
 ## [2.0.0-alpha] - 2023-06-01
 - update gltf-transform dependency to 4.x
 - change: ensure ETC1S and UASTC textures are power of 2 

package/dist/___config.json

@@ -0,0 +1,3 @@
+{
+    "usecase": "test"
+}

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@needle-tools/gltf-build-pipeline",
-    "version": "2.0.0-alpha",
+    "version": "2.1.0-alpha.1",
     "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
     "main": "index.mjs",
     "type": "module",

package/dist/src/cache/cache.d.ts

@@ -48,6 +48,7 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
 /**
  * Get the hash for a property or an object.
  * @param property The property to get the hash for.
+ * @param level The current level of recursion.
  * @returns The hash as a number.
  */
 export declare function getHash(property: ExtensibleProperty | object, level?: number): number;

package/dist/src/cache/cache.js

@@ -167,6 +167,7 @@ export function getKeyWithHash(property) {
 /**
  * Get the hash for a property or an object.
  * @param property The property to get the hash for.
+ * @param level The current level of recursion.
  * @returns The hash as a number.
  */
 export function getHash(property, level = 0) {

package/dist/src/cli/index.js

@@ -45,11 +45,12 @@ Compress a single glTF file. The input file is compressed and written to the out
         totalFileSizeInMB: 0,
         totalFileSizeInMBBefore: 0,
     };
-    let config = getConfig(options.config?.toString());
+    const file = args.input.toString();
+    let config = getConfig(options.config?.toString(), file);
     if (options.usecase) {
         logger.info("Using usecase: " + options.usecase);
         config = { ...createConfig(options.usecase.toString()), ...config, };
-        if (config.usecase != "unknown" && config.usecase != options.usecase.toString()) {
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
             logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
         }
         else {
@@ -57,7 +58,6 @@ Compress a single glTF file. The input file is compressed and written to the out
         }
     }
     const debug = options.debug === true;
-    const file = args.input.toString();
     const isDirectory = statSync(file).isDirectory();
     const outDir = isDirectory ? file : dirname(file);
     let outputIsDirectory = false;
@@ -98,11 +98,11 @@ Compress a single glTF file. The input file is compressed and written to the out
             process.exit(ERROR_CODES.PACKING_FAILED);
         }
     });
-    printStats(stats);
+    printStats(stats, logger, config);
     console.log("Stats: ", options);
     if (options.stats) {
         const statsFile = (isDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
-        writeFileStatsToFile(file_stats, statsFile);
+        writeFileStatsToFile(config, file_stats, statsFile);
     }
 })
     .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
@@ -126,18 +126,18 @@ Each version will be compressed and written to the output directory.
         totalFileSizeInMB: 0,
         totalFileSizeInMBBefore: 0,
     };
-    let config = getConfig(options.config?.toString());
+    const file = args.input.toString();
+    let config = getConfig(options.config?.toString(), file);
     if (options.usecase) {
         logger.info("Using usecase: " + options.usecase);
         config = { ...createConfig(options.usecase.toString()), ...config, };
-        if (config.usecase != "unknown" && config.usecase != options.usecase.toString()) {
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
             logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
         }
         else {
             config.usecase = options.usecase.toString();
         }
     }
-    let file = args.input.toString();
     const isDirectory = statSync(file).isDirectory();
     const outDir = isDirectory ? file : dirname(file);
     const file_stats = [];
@@ -175,10 +175,10 @@ Each version will be compressed and written to the output directory.
             process.exit(ERROR_CODES.PACKING_FAILED);
         }
     });
-    printStats(stats);
+    printStats(stats, logger, config);
     if (options.stats) {
         const statsFile = (isDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
-        writeFileStatsToFile(file_stats, statsFile);
+        writeFileStatsToFile(config, file_stats, statsFile);
     }
 });
 program.run();

package/dist/src/config/index.d.ts

@@ -1,4 +1,4 @@
-declare type Usecase = "unknown" | "product" | "world";
+declare type Usecase = "default" | "product" | "world";
 export declare const UsecaseOptions: string[];
 /**
  * The config object contains information about how a file should be processed.
@@ -6,6 +6,9 @@ export declare const UsecaseOptions: string[];
 export type Config = {
     /** High-level configuration */
     usecase: Usecase;
+    textures?: {
+        lods: boolean;
+    };
 };
 /**
  * Create a config object with the given usecase.
@@ -39,5 +42,6 @@ export declare function createConfig(usecase: string): {
  *
  *
  */
-export declare function getConfig(path?: string): Config;
+export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string): Config;
+export declare function searchConfig(directory: string): Config | null;
 export {};

package/dist/src/config/index.js

@@ -1,7 +1,8 @@
-import { existsSync, readFileSync } from "fs";
-export const UsecaseOptions = ["product"];
+import { existsSync, readFileSync, statSync } from "fs";
+import path, { dirname } from "path";
+export const UsecaseOptions = ["product", "world"];
 const defaultConfig = {
-    usecase: "unknown",
+    usecase: "default",
 };
 /**
  * Create a config object with the given usecase.
@@ -40,49 +41,46 @@ export function createConfig(usecase) {
  *
  *
  */
-export function getConfig(path) {
-    if (path != null) {
-        if (!existsSync(path)) {
-            throw new Error(`No config found at \"${path}\"`);
+export function getConfig(configPath, fallbackSearchDirectory) {
+    if (configPath != null) {
+        if (!existsSync(configPath)) {
+            throw new Error(`No config found at \"${configPath}\"`);
+        }
+        const res = resolveConfig(configPath, readFileSync(configPath, "utf8"));
+        if (res) {
+            return res;
+        }
+        else {
+            throw new Error(`Failed parsing config at \"${configPath}\"`);
         }
-        return resolveConfig(path, readFileSync(path, "utf8"));
     }
     if (_cachedConfig !== undefined) {
         return _cachedConfig;
     }
     // Resolve the config once and cache it
-    const result = internalResolveConfig();
-    if (result) {
-        console.log("INFO: Using optimization config with usecase: " + result.usecase);
-        _cachedConfig = result;
+    // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
+    if (fallbackSearchDirectory) {
+        const result = searchConfig(fallbackSearchDirectory);
+        if (result) {
+            _cachedConfig = { ...defaultConfig, ...result };
+            console.log("INFO: Using config with usecase: " + _cachedConfig.usecase);
+        }
     }
-    else {
-        console.log("INFO: No optimization config found. Using default config.");
+    // If no config was found, use the default config
+    if (!_cachedConfig) {
+        console.log("INFO: No config found. Using default config.");
         _cachedConfig = defaultConfig;
     }
     return _cachedConfig;
 }
 let _cachedConfig = undefined;
-function internalResolveConfig() {
-    // Check if a config path is passed as an argument
-    const args = process.argv.slice(2);
-    for (let i = 0; i < args.length; i++) {
-        if (args[i] === "--config") {
-            if (i + 1 < args.length) {
-                const configPath = args[i + 1];
-                if (existsSync(configPath)) {
-                    const content = readFileSync(configPath, "utf8");
-                    return resolveConfig(configPath, content);
-                }
-                throw new Error(`Config file not found at path configured by CLI --config argument: \"${configPath}\"`);
-            }
-        }
+export function searchConfig(directory) {
+    if (statSync(directory).isFile()) {
+        directory = dirname(directory);
     }
-    // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
-    const workingDir = process.cwd();
-    let directory = workingDir.replaceAll("\\", "/");
+    directory = directory.replaceAll("\\", "/");
     while (directory.length > 0) {
-        const configPath = `${directory}/needle.config.json`;
+        const configPath = path.join(directory, "needle.config.json");
         if (existsSync(configPath)) {
             const content = readFileSync(configPath, "utf8");
             const config = resolveConfig(configPath, content);
@@ -103,16 +101,16 @@ function resolveConfig(filename, content) {
     if (filename.endsWith("needle.config.json")) {
         try {
             const config = JSON.parse(content);
-            if (config.optimization) {
-                console.log("INFO: Found optimization config at \"" + filename + "\".");
+            if (config.gltf) {
+                console.log("INFO: Found \"gltf\" config at \"" + filename + "\".");
                 // If the optimization config is a string, it's the usecase
-                if (typeof config.optimization === "string") {
-                    return createConfig(config.optimization);
+                if (typeof config.gltf === "string") {
+                    return createConfig(config.gltf);
                 }
-                return config.optimization;
+                return config.gltf;
             }
             else {
-                console.log("INFO: No optimization config found in needle.config.json. Using default config.");
+                console.log("INFO: No \"gltf\" config found in needle.config.json. Using default config.");
                 return null;
             }
         }

package/dist/src/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -624,6 +624,27 @@ class JsonPointerHandler {
                             pointer?.register(obj, key, fullPath);
                         }
                     }
+                    if (value.startsWith("/animations/")) {
+                        if (this.debug)
+                            console.log("Read pointer", value);
+                        const index = parseInt(value.substring("/animations/".length));
+                        const object = context.animations[index];
+                        if (object) {
+                            let pointer = this.map.get(value);
+                            if (!pointer) {
+                                // TODO: resolve animations + we need to set the reference properly so we can use prune. See https://linear.app/needle/issue/NE-4982
+                                // const definition = context.jsonDoc.json.animations![index];
+                                // const res = new AnimationIndexResolver(this.document, object, definition);
+                                // res.debug = this.debug;
+                                // res.init(context);
+                                // pointer = new JsonPointer("/animations/", res, value);
+                                // pointer.debug = this.debug;
+                                // this.map.set(value, pointer);
+                                prop.setReference(value, object);
+                            }
+                            pointer?.register(obj, key, fullPath);
+                        }
+                    }
                 }
                 break;
             case 'number':
@@ -682,7 +703,7 @@ class JsonPointerHandler {
                         pointer.debug = this.debug;
                         pointer.register(obj, key, fullPath);
                         this.map.set(dictKey, pointer);
-                        prop.setReference(dictKey, object);
+                        prop.setReference(dictKey, object, { channels: TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A });
                     }
                 }
                 break;

package/dist/src/extensions/NEEDLE_progressive/NEEDLE_progressive.d.ts

@@ -12,8 +12,12 @@ type LodInfo = {
     /**  Hash of the LOD file. It has been generated using the file content and settings */
     hash: string;
 };
+export type ProgressiveTextureLodEntry = LodInfo & {
+    width: number;
+    height: number;
+};
 export type ProgressiveTextureModel = ProgressiveLoadingInfo & {
-    lods?: Array<LodInfo & {}>;
+    lods?: Array<ProgressiveTextureLodEntry>;
 };
 export type ProgressiveMeshModel = ProgressiveLoadingInfo & {
     /** LOD0 vertex density. TODO: remove this since we now have a LODs array. This is just for supporting older engine versions */

package/dist/src/scripts/pack-gltf.js

@@ -9,7 +9,6 @@ import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
 import { createNewFilestats, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
-import { getConfig } from '../config/index.js';
 /** Compress a glTF file
  */
 export async function packGLTF(inputFile, outputFile, options) {
@@ -42,7 +41,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         packedFiles.add(inputFile);
     }
     try {
-        const config = options.config || getConfig();
+        const config = options.config;
         const prevSizeInMB = statSync(inputFile).size / 1024 / 1024;
         logger.info(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
         const startTime = Date.now();
@@ -85,7 +84,10 @@ export async function packGLTF(inputFile, outputFile, options) {
         const isProgressiveAsset = isLOD(inputFile);
         // TODO: progressive mesh LODs should not be pruned right now because it will remove e.g. UVs from meshes (and probably blend shapes too) (so we just not do it for any progressive asset) 
         if (!isProgressiveAsset) {
-            transforms.push(prune());
+            transforms.push(prune({
+                // Pruning animations is not supported yet (e.g. if the animation is only referenced by a component)
+                propertyTypes: ['MESH', 'MATERIAL', 'TEXTURE']
+            }));
         }
         await document.transform(...transforms);
         if (!outputFile)

package/dist/src/transforms/needle_mesh_transform.js

@@ -16,10 +16,10 @@ async function applyMeshCompression(document, transform_context, context) {
         const rootModel = rootExtension.model;
         if (rootModel.compression) {
             if (rootModel.compression === "none") {
-                console.log(`[${NAME}]: NONE → will not apply mesh compression`);
+                document.getLogger().info(`[${NAME}]: NONE → will not apply mesh compression`);
                 return;
             }
-            console.log(`[${NAME}]: ${rootModel.compression.toUpperCase()} → will apply ${rootModel.compression} mesh compression`);
+            document.getLogger().info(`[${NAME}]: ${rootModel.compression.toUpperCase()} → will apply ${rootModel.compression} mesh compression`);
             if (rootModel.compression === "draco") {
                 const transform = draco();
                 return transform(document, transform_context);
@@ -31,13 +31,13 @@ async function applyMeshCompression(document, transform_context, context) {
         }
     }
     // If nothing is configured apply meshopt compression as long as we're running gltf-transform 3.x
-    if (context.file.endsWith(".vrm")) {
-        console.log(`[${NAME}]: VRM → will apply meshopt mesh compression`);
-        const transform = meshopt({ encoder: MeshoptEncoder });
-        return transform(document, transform_context);
-    }
+    // if (context.file.endsWith(".vrm")) {
+    //     document.getLogger().info(`[${NAME}]: VRM → will apply meshopt mesh compression`);
+    //     const transform = meshopt({ encoder: MeshoptEncoder });
+    //     return transform(document, transform_context);
+    // }
     // by default we still want to run draco if the root doesnt have a compression model configured
-    console.log(`[${NAME}]: No compression model configured → will apply draco mesh compression`);
+    console.log(`[${NAME}]: → will apply draco mesh compression (default)`);
     return draco()(document, transform_context);
 }
 async function simplifierTransform(document, _context, _options) {

package/dist/src/transforms/needle_progressive.js

@@ -1,11 +1,11 @@
-import { BufferUtils, Document, NodeIO, PropertyType } from '@gltf-transform/core';
-import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots, sparse } from '@gltf-transform/functions';
+import { BufferUtils, Document, NodeIO, PropertyType, TextureChannel, Verbosity } from '@gltf-transform/core';
+import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureChannels, listTextureSlots, sparse } from '@gltf-transform/functions';
 import draco3d from 'draco3dgltf';
 import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive } from "../extensions/NEEDLE_progressive/index.js";
 import { createOpaqueExtension, currentGenerator, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
 import path, { basename, resolve } from 'path';
 import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
-import { addMetaToCache, addToCache, getHash, getKeyWithHash, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
+import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
 import { detectTextureCompressionMode, foreachGLTF, generateGuid, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
@@ -55,18 +55,13 @@ async function onProcess(file, opts) {
         });
         let document = await io.read(file);
         if (document) {
-            if (opts.debug === true) {
-                if (file.includes(".debug.")) {
-                    console.log("Skip debug file: " + file);
-                    return;
-                }
-            }
+            const logger = document.getLogger();
+            //@ts-ignore
+            logger.verbosity = opts.debug ? Verbosity.DEBUG : Verbosity.WARN;
             let smallesTextureSize = 128;
             if (opts.config.usecase === "product") {
                 smallesTextureSize = 512;
             }
-            //@ts-ignore
-            document.getLogger().verbosity = 0;
             const transforms = [
                 needle_asset(),
                 make_progressive_textures(file, { size: [smallesTextureSize, smallesTextureSize] }, opts),
@@ -77,8 +72,8 @@ async function onProcess(file, opts) {
                 transforms.push(make_progressive_meshes(file, {}, opts));
             }
             else
-                console.warn("WARN: Progressive meshes are not supported for VRM files yet");
-            console.log("→ Transforming: " + file);
+                logger.warn("WARN: Progressive meshes are not supported for VRM files yet");
+            logger.info("→ Transforming: " + file);
             await document.transform(...transforms);
             let filename = file;
             if (opts.debug === true) {
@@ -91,7 +86,7 @@ async function onProcess(file, opts) {
             }
             await writeNodeIO(io, filename, document);
             opts.results.push(filename);
-            console.log("← Transforming done: " + filename);
+            logger.info("← Transforming done: " + filename);
         }
     }
     catch (err) {
@@ -108,7 +103,6 @@ export function createTransform(name, fn) {
 }
 const NAME = 'NEEDLE_progressive';
 export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DEFAULTS, opts) {
-    console.log("Make progressive: " + filePath);
     const startTime = Date.now();
     const filename = basename(filePath);
     const options = { ...TEXTURE_RESIZE_DEFAULTS, ..._options };
@@ -117,6 +111,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
     const textureLodBasePath = directory + "/";
     return createTransform(NAME, async (doc) => {
         const logger = doc.getLogger();
+        if (isLOD(filePath)) {
+            logger.info("Progressive Image Transform: skip lod at " + filePath);
+            return Promise.resolve();
+        }
         let index = 0;
         for (const texture of doc.getRoot().listTextures()) {
             const textureIndex = index++;
@@ -159,8 +157,6 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 logger.warn(`[${NAME}] Skipping texture[${textureIndex}], settings are invalid (maxSize: ${settings.maxSize})`);
                 continue;
             }
-            options.size[0] = settings.maxSize;
-            options.size[1] = settings.maxSize;
             const existingExtension = texture.getExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME);
             if (existingExtension !== undefined && existingExtension !== null) {
                 logger.debug(`[${NAME}] Skipping, texture is already progressive ${textureIndex}`);
@@ -172,25 +168,62 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 continue;
             }
             const textureGuid = settings.guid?.length ? settings.guid : generateGuid(filename + "_texture_" + index.toString());
-            const outPath = createTextureLod(doc, filePath, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
-            if (!outPath) {
-                continue;
+            /** This is the size of the texture that is embedded in the glTF. No LOD should be smaller than this */
+            const maxSize = settings.maxSize;
+            /** This is the current texture's size  */
+            const currentSize = Math.min(...texture.getSize());
+            const LOD_LEVEL = new Array();
+            LOD_LEVEL.push({ size: currentSize });
+            let generateLods = opts.config?.usecase === "world";
+            // If lods are enabled in the config
+            generateLods ||= opts.config?.textures?.lods === true;
+            // Ensure that texture lods are not disabled by { textures: { lods: false } }
+            generateLods &&= opts.config?.textures?.lods !== false;
+            if (generateLods) {
+                const LEVEL = 6;
+                let lastSize = currentSize;
+                for (let i = 1; i < LEVEL; i++) {
+                    const size = lastSize * .5;
+                    if (size <= maxSize)
+                        break;
+                    lastSize = size;
+                    LOD_LEVEL.push({ size: size });
+                }
+            }
+            const results = new Array();
+            for (let i = 0; i < LOD_LEVEL.length; i++) {
+                const level = LOD_LEVEL[i];
+                const lodResult = await createTextureLod({
+                    doc,
+                    file: filePath,
+                    basePath: textureLodBasePath,
+                    level: i,
+                    texture,
+                    slots,
+                    guid: textureGuid,
+                    textureIndex,
+                    maxSize: level.size
+                }, opts);
+                if (!lodResult) {
+                    continue;
+                }
+                const { 0: width, 1: height } = lodResult.texture.getSize();
+                const outpath = lodResult?.outpath;
+                const relName = path.relative(directory, outpath);
+                results.push({
+                    path: relName,
+                    hash: lodResult.hash,
+                    width,
+                    height,
+                });
+                opts.results.push(outpath);
             }
-            opts.results.push(outPath);
-            const { key, hash } = getKeyWithHash(texture);
-            const relName = path.relative(directory, outPath);
+            const key = getKey(texture);
             // TODO: this path should be added to the NEEDLE dependencies extension
             const model = {
                 guid: textureGuid,
-                lods: [
-                    {
-                        hash: hash.toString(),
-                        path: relName,
-                    }
-                ],
+                lods: results,
             };
-            // for backwards compatibility - this should be removed in the future
-            model["uri"] = relName;
             texture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createTexture(model));
             if (opts.useCache) {
                 const cached = tryGetFromCache(key);
@@ -200,6 +233,9 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                     continue;
                 }
             }
+            options.size[0] = settings.maxSize;
+            options.size[1] = settings.maxSize;
+            logger.debug(`[${NAME}] Resizing embedded texture[${index}] to ${options.size[0]}x${options.size[1]}`);
             await resizeTexture(options, texture, logger, uri || name, slots);
             if (opts?.useCache === false) {
             }
@@ -209,14 +245,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
         logger.debug(`Progressive transform finished in ${((Date.now() - startTime) / 1000).toFixed(1)} sec.`);
     });
 }
-function createTextureLod(doc, file, basePath, level, texture, slots, guid, textureIndex) {
-    // const imageBytes = texture.getImage();
-    // if (imageBytes) {
-    // 	// write bytes
-    // 	const outPath = basePath + guid + "_" + level + ".png";
-    // 	writeFileSync(outPath, imageBytes);
-    // 	return outPath;
-    // }
+async function createTextureLod(args, opts) {
+    const { doc, file, basePath, level, texture, guid, textureIndex } = args;
+    let slots = args.slots;
+    const logger = doc.getLogger();
     const newDoc = new Document();
     writeNeedleAsset(newDoc);
     const scene = newDoc.createScene();
@@ -227,14 +259,25 @@ function createTextureLod(doc, file, basePath, level, texture, slots, guid, text
     newTexture.setImage(img);
     newTexture.setURI(texture.getURI());
     newTexture.setMimeType(texture.getMimeType());
+    if (args.maxSize && args.maxSize >= 16) {
+        const currentSize = newTexture.getSize();
+        if (currentSize[0] > args.maxSize && currentSize[1] > args.maxSize) {
+            await resizeTexture({ size: [args.maxSize, args.maxSize] }, newTexture, logger, texture.getURI(), slots);
+        }
+    }
     if (!slots?.length) {
-        console.warn(`WARN: No material slots found for texture at index ${textureIndex} (name: \"${texture.getName()}\") → will assume slot \"baseColorTexture\"`);
+        logger.warn(`WARN: No material slots found for texture at index ${textureIndex} (name: \"${texture.getName()}\") → will assume slot \"baseColorTexture\"`);
         slots = ["baseColorTexture"];
     }
     let slot_index = 0;
     // create material adding the texture to the same slot
     for (const slot of slots) {
-        const newMaterial = newDoc.createMaterial();
+        const newMaterial = newDoc.createMaterial(newTexture.getName());
+        const channels = listTextureChannels(texture);
+        // Ensure that we use the alpha channel if it is present
+        if (channels.includes(TextureChannel.A)) {
+            newMaterial.setAlphaMode("BLEND");
+        }
         switch (slot) {
             case "baseColorTexture":
                 newMaterial.setBaseColorTexture(newTexture);
@@ -252,7 +295,14 @@ function createTextureLod(doc, file, basePath, level, texture, slots, guid, text
                 newMaterial.setEmissiveTexture(newTexture);
                 break;
             default:
-                console.warn("WARN: Unknown texture slot: " + slot + " for texture " + textureIndex + " → can not load progressively");
+                if (slot.startsWith("/textures/")) {
+                    // Ensure the texture is using the alpha channel since we don't know what slots the original texture was used for
+                    // see https://linear.app/needle/issue/NE-4985
+                    newMaterial.setAlphaMode("BLEND");
+                    newMaterial.setBaseColorTexture(newTexture);
+                    continue;
+                }
+                logger.warn(`WARN: Unknown texture slot: ${slot} for texture ${textureIndex} → can not load progressively`);
                 return null;
         }
         // Make sure the material is used
@@ -260,16 +310,23 @@ function createTextureLod(doc, file, basePath, level, texture, slots, guid, text
     }
     const compressionSettings = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
     if (compressionSettings !== undefined && compressionSettings !== null && compressionSettings.getExtensionDefinition) {
-        console.log("> Pass compression extension to new texture");
+        logger.info("> Pass compression extension to new texture");
         const newExt = newDoc.createExtension(NEEDLE_compression_texture);
         const def = compressionSettings.getExtensionDefinition();
-        def.mode = detectTextureCompressionMode(slots);
+        switch (opts.config.usecase) {
+            // For world scenarios we use ETC1S
+            case "world":
+                def.mode = "ETC1S";
+                break;
+            case "product":
+                def.mode = detectTextureCompressionMode(slots);
+                def.quality = 100;
+                break;
+            default:
+                def.mode = detectTextureCompressionMode(slots);
+                break;
+        }
         def.slot = slots[0];
-        // for VRMS we just use webp
-        // if (file?.toLocaleLowerCase().endsWith(".vrm")) {
-        // 	def.mode = "webp";
-        // 	def.quality = 70;
-        // }
         const newProp = newExt.create(def);
         newTexture.setExtension(NEEDLE_compression_texture.EXTENSION_NAME, newProp);
     }
@@ -284,9 +341,13 @@ function createTextureLod(doc, file, basePath, level, texture, slots, guid, text
     const imageName = "image_" + level + "_" + guid;
     const filename = imageName + ext;
     const outName = basePath + filename;
-    console.log(`INFO: Write texture LOD ${level} to \"${filename}\"`);
+    logger.info(`Write texture LOD ${level} to \"${filename}\"`);
     newIo.write(outName, newDoc);
-    return outName;
+    return {
+        outpath: outName,
+        texture: newTexture,
+        hash: getHash(newTexture).toString(),
+    };
 }
 function createMeshForMaterial(doc, mat, i) {
     const buffer = doc.getRoot().listBuffers()[0];
@@ -369,6 +430,7 @@ async function resizeTexture(options, texture, logger, uri, slots) {
         dstWidth = Math.floor(dstWidth * (maxHeight / dstHeight));
         dstHeight = maxHeight;
     }
+    logger.info("> Resize texture to " + dstWidth + "x" + dstHeight + " (from " + srcWidth + "x" + srcHeight + ")");
     // https://sharp.pixelplumbing.com/api-resize
     const buffer = await sharp(texture.getImage())
         .resize(dstWidth, dstHeight, {
@@ -378,26 +440,30 @@ async function resizeTexture(options, texture, logger, uri, slots) {
     })
         .toBuffer();
     texture.setImage(BufferUtils.toView(buffer));
+    return {
+        width: dstWidth,
+        height: dstHeight,
+    };
 }
 export function make_progressive_meshes(filePath, _options, opts) {
     const directory = path.dirname(filePath);
     // const fileNameWithoutExtension = path.basename(filePath, path.extname(filePath));
     const outputDirectory = directory + "/";
     return createTransform("PROGRESSIVE_MESHES", async (doc, _context) => {
+        const logger = doc.getLogger();
         // this is to ensure we're not creating progressive meshes for engine versions that don't support loading them yet
         // the generator should include the version or the app that produced the GLB too (needle engine repo: 2c00d0e304e205cf9d4ac0499faabb92a706a5c4)
         if (currentGenerator == "Needle Engine") {
-            console.log("WARN: Skip creating mesh LODs - Unsupported generator found... Please update your editor integration");
+            logger.warn("WARN: Skip creating mesh LODs - Unsupported generator found... Please update your editor integration");
             return;
         }
-        if (isMeshLOD(filePath)) {
-            console.log("Progressive Mesh Transform: skip lod at " + filePath);
+        if (isLOD(filePath)) {
+            logger.info("Progressive Mesh Transform: skip lod at " + filePath);
             return Promise.resolve();
         }
         await MeshoptSimplifier.ready;
-        const logger = doc.getLogger();
         const filename = path.basename(filePath);
-        console.log(`[${NAME}]: Begin generating progressive meshes for \"${filename}\"`);
+        logger.info(`[${NAME}]: Begin generating progressive meshes for \"${filename}\"`);
         let index = 0;
         // the lowest LOD (last entry) will be saved in the original file
         let MAX_LODS = 6;
@@ -433,7 +499,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
         for (const mesh of meshes) {
             await processMesh(mesh, { filename });
         }
-        console.log(`[${NAME}]: Finished generating progressive meshes for \"${filename}\"`);
+        logger.info(`[${NAME}]: Finished generating progressive meshes for \"${filename}\"`);
         async function processMesh(mesh, args) {
             /** The size and vertex count of the original mesh */
             const originalMeshInformation = getMeshInformation(mesh);
@@ -474,7 +540,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
             /** async simplification tasks... */
             const tasks = new Array();
             if (needsToGenerateLODs) {
-                console.log(`[${NAME}] Begin generating mesh LODs for \"${mesh.getName()}\"`);
+                logger.info(`[${NAME}] Begin generating mesh LODs for \"${mesh.getName()}\"`);
                 // produce LODS, skip the last level because it will be saved in the original file
                 for (let level = 0; level < LODS.length - 1; level++) {
                     const lod = LODS[level];
@@ -510,11 +576,11 @@ export function make_progressive_meshes(filePath, _options, opts) {
                     let shouldOmitLevel = false;
                     // if the mesh size on disc is identical to the last level, we can omit this level
                     if (sizeOnDisc <= 0) {
-                        console.log(`[${NAME}] Flag mesh LOD ${level}: size on disc is 0 → level will be omitted (if this mesh is is taken from cache it can mean that the LOD level was omitted before). \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
+                        logger.info(`[${NAME}] Flag mesh LOD ${level}: size on disc is 0 → level will be omitted (if this mesh is is taken from cache it can mean that the LOD level was omitted before). \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                         shouldOmitLevel = true;
                     }
                     else if (diff == 0) {
-                        console.log(`[${NAME}] Flag mesh LOD ${level}: size on disc did not change compared to previous level → level will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
+                        logger.info(`[${NAME}] Flag mesh LOD ${level}: size on disc did not change compared to previous level → level will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                         shouldOmitLevel = true;
                     }
                     else if (!shouldOmitLevel) {
@@ -557,7 +623,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
                 if (data != null) {
                     clonedData.primitives.push(data);
                 }
-                console.log(`[${NAME}] Simplify mesh[${index}] primitive[${primitiveIndex}] (embedded)`);
+                logger.info(`[${NAME}] Simplify mesh[${index}] primitive[${primitiveIndex}] (embedded)`);
                 const lastLod = LODS[lodLevelInFile];
                 // TODO: this should be run after simplification but then we need to change the runtime to use the LOD 0 density
                 const density = calculateMeshDensity(prim);
@@ -575,7 +641,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
             }
             else if (difference < 32 || diff01 <= 0.05) {
                 // the lowest LOD has the same vertex count as the original mesh → we don't need to write the extension and can also delete previously generated LODs
-                console.log(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length} (embedded): vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove LOD. \"${mesh.getName()}\"`);
+                logger.info(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length} (embedded): vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove LOD. \"${mesh.getName()}\"`);
                 try {
                     for (const lodInfo of generatedLODInfos) {
                         rmSync(outputDirectory + "/" + lodInfo.path);

package/dist/src/transforms/needle_toktx.js

@@ -96,7 +96,7 @@ export class NeedleToktxContext {
             const doc = this.document;
             const logger = doc.getLogger();
             const slots = listTextureSlots(texture);
-            const channels = getTextureChannelMask(texture);
+            let channels = getTextureChannelMask(texture);
             // const textureLabel =
             //     texture.getURI() ||
             //     texture.getName() ||
@@ -175,6 +175,10 @@ export class NeedleToktxContext {
                 }
             }
             const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
+            // TODO: should we ensure that the texture is using all channels? Unused channels are discarded anyways. The issue we have here is that textures that are solely used from pointers have unknown channels. We register them with RGBA when generating progressive assets as well as assign texture json pointers /textures/ to
+            // Alpha blended materials to make sure RGBA channels are used when progressive GLBs are compressed (in needle_progressive 377) 
+            // https://linear.app/needle/issue/NE-4985
+            // channels = TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A;
             //@ts-ignore
             const toktxParams = createParams(texture, slots, channels, srcSize, logger, this.numTextures, { ...defaults, mode, quality, compression });
             await fs.writeFile(inPath, Buffer.from(srcImage));

package/dist/src/utils/fileutils.js

@@ -57,7 +57,7 @@ export async function foreachGLTF(args, callback) {
         }
     }
     if (context.files.length <= 0) {
-        console.log("WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory. This path may be relative to " + initDir);
+        console.log("WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory. This path may be relative to \"" + initDir + "\"");
     }
     else {
         for (const file of context.files) {

package/dist/src/utils/stats.d.ts

@@ -1,5 +1,6 @@
 import { Logger } from "@donmccurdy/caporal";
 import { Document } from "@gltf-transform/core";
+import { Config } from "../config/index.js";
 export type Stats = {
     startTime: number;
     totalGPUMemoryInMB: number;
@@ -7,7 +8,7 @@ export type Stats = {
     totalFileSizeInMB: number;
     totalFileSizeInMBBefore: number;
 };
-export declare function printStats(stats: Stats, logger?: Logger): void;
+export declare function printStats(stats: Stats, logger: Logger, config: Config): void;
 export type FileStats = {
     filename: string;
     /** Size in bytes */
@@ -25,4 +26,4 @@ export type FileStats = {
  * Create a new filestats object from a file and a document
  */
 export declare function createNewFilestats(file: string, document: Document, stats: Pick<FileStats, "gpu_memory">): FileStats;
-export declare function writeFileStatsToFile(filestats: FileStats[], output_path: string): void;
+export declare function writeFileStatsToFile(config: Config, filestats: FileStats[], output_path: string): void;

package/dist/src/utils/stats.js

@@ -1,10 +1,10 @@
 import { getVersion } from "./version.js";
 import { getFileName } from "./fileutils.js";
 import { statSync, writeFileSync } from "fs";
-export function printStats(stats, logger) {
+export function printStats(stats, logger, config) {
     const duration = Date.now() - stats.startTime;
     const durationInMinutes = duration / 1000 / 60;
-    const msg = `[Needle Build Pipeline] v${getVersion()} — Finished packing ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
+    const msg = `[Needle Build Pipeline] v${getVersion()} — Finished ${config?.usecase} compression ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
     if (logger)
         logger.info(msg);
     else
@@ -34,7 +34,7 @@ export function createNewFilestats(file, document, stats) {
     filestats.triangle_count = filestats.vertex_count / 3;
     return filestats;
 }
-export function writeFileStatsToFile(filestats, output_path) {
+export function writeFileStatsToFile(config, filestats, output_path) {
     if (output_path == undefined) {
         console.error("No stats output path provided");
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.0.0-alpha",
+  "version": "2.1.0-alpha.1",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",

package/scripts/pack-gltf.mjs

@@ -59,7 +59,7 @@ if (noToktx)
 const packedFiles = new Set();
 
 /**
- * @type {import("../dist/index.js").Stats}
+ * @type {import("../dist/src/index.js").Stats}
  */
 const stats = {
     startTime: Date.now(),