@needle-tools/gltf-build-pipeline 1.6.0-alpha.7 → 2.1.0-alpha

package/CHANGELOG.md

@@ -4,6 +4,45 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.1.0-alpha] - 2023-06-03
+- add: compression config support for high-level usecases that can be passed via commandline arguments or via config. Available options are currently `default`, `world` and `product`.  
+- add: texture LODs can now be generated by passing in the path to a compression config. `--config <path/to/config.json>` 
+- change: VRMs now use draco mesh compression by default 
+
+### Config Examples
+  - `default` - use default compression. When not explictly configured textures will be compressed with ETC1S
+  - `product` - when not explictly configured textures will use UASTC for `metallicRoughnessTexture` and `normalTexture`   
+  - `world` - generate texture LODs when enabled (and not explictly disabled by `textures: { lods: false }`)
+  ```json
+  {
+    "usecase": "default" // optional
+    "textures" : { 
+        "lods": true
+    }
+  }
+  ```
+  The config can also be part of a `needle.config.json`:
+  ```json
+  {
+    ... regular needle.config
+    "gltf": {
+        "usecase": "default" // optional
+        "textures": {
+            "lods": true
+        }
+    }
+  }
+  ```
+
+## [2.0.0-alpha] - 2023-06-01
+- update gltf-transform dependency to 4.x
+- change: ensure ETC1S and UASTC textures are power of 2 
+
+- CLI: commands can be run with `needle-gltf <command>`
+- CLI: add `--cache` to allow disabling cache with `--cache False` (default true)
+- CLI: add `--stats` to produce `.needle.stats.json` (default false)
+- CLI: change `--output` to be the optional second argument
+
 ## [1.6.0-alpha.7] - 2023-05-30
 - fix: npx cli command return wrong error due to `packGLTF` not returning a booelan
 - fix: mjs script import paths

package/README.md

@@ -8,6 +8,24 @@ Includes support for generating LOD quality level for textures and meshes that c
 - [Example on Github](https://github.com/needle-engine/progressive-meshes-sample)
 
 
+## Usage
+
+In general all files that have been processed by this package are valid glTF/GLB files.    
+The glTF Build Pipeline package is meant to be used as a last step before delivery.   
+That means:
+- files are meant to be loaded e.g. in a web runtime like three.js 
+- files are not to be opened in Blender again to be further edited    
+
+To load progressively enhanced glTF files in any three.js based engine see examples at [@needle-tools/gltf-progressive](https://www.npmjs.com/package/@needle-tools/gltf-progressive)
+
+### Installation
+
+`npx @needle-tools/gltf-build-pipeline@latest`
+
+### Commands
+- `gltf-build-pipeline help` - use help to see options for each command
+- `gltf-build-pipeline compress <input>`
+- `gltf-build-pipeline transform <input>`
 
 # Contact ✒️
 <b>[🌵 needle — tools for creators](https://needle.tools)</b> • 

package/dist/___config.json

@@ -0,0 +1,3 @@
+{
+    "usecase": "test"
+}

package/dist/package.json

@@ -1,30 +1,51 @@
 {
     "name": "@needle-tools/gltf-build-pipeline",
-    "version": "1.6.0-alpha.7",
+    "version": "2.1.0-alpha",
     "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
     "main": "index.mjs",
     "type": "module",
     "readme": "README.md",
+    "author": {
+        "name": "Needle",
+        "email": "help@needle.tools",
+        "url": "https://needle.tools/"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/needle-tools/needle-gltf-extensions.git"
+    },
     "bin": {
-        "gltf-build-pipeline": "./dist/src/cli/index.js"
+        "gltf-build-pipeline": "./dist/src/cli/index.js",
+        "needle-gltf": "./dist/src/cli/index.js"
     },
     "scripts": {
         "start": "node scripts/index.mjs",
         "dev": "npm-watch compile",
+        "compile": "tsc || exit 1",
+        "clear-caches": "node scripts/clear-caches.mjs",
+        "limit-caches": "node scripts/limit-caches.mjs",
         "transform": "node scripts/index.mjs",
         "transform:make-progressive": "node scripts/make-progressive.mjs",
         "transform:pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
         "pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
-        "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs",
-        "clear-caches": "node scripts/clear-caches.mjs",
-        "limit-caches": "node scripts/limit-caches.mjs",
-        "compile": "tsc || exit 1"
+        "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs"
+    },
+    "devDependencies": {
+        "@img/sharp-win32-x64": "^0.33.4",
+        "@types/node": "^18.7.23",
+        "npm-watch": "^0.13.0",
+        "typescript": "^4.9.5"
+    },
+    "optionalDependencies": {
+        "@img/sharp-linux-arm": "^0.33.4",
+        "@img/sharp-win32-x64": "^0.33.4"
     },
     "dependencies": {
         "@donmccurdy/caporal": "~0.0.10",
-        "@gltf-transform/core": "^3.10.1",
-        "@gltf-transform/extensions": "^3.10.1",
-        "@gltf-transform/functions": "^3.10.1",
+        "@gltf-transform/cli": "^4.0.1",
+        "@gltf-transform/core": "^4.0.1",
+        "@gltf-transform/extensions": "^4.0.1",
+        "@gltf-transform/functions": "^4.0.1",
         "command-exists": "^1.2.9",
         "command-line-args": "^5.2.1",
         "draco3dgltf": "^1.5.3",
@@ -36,7 +57,7 @@
         "ndarray-pixels": "^2.0.1",
         "p-limit": "3.1.0",
         "semver": "^7.3.7",
-        "sharp": "^0.32.6",
+        "sharp": "^0.33.4",
         "tmp": "^0.2.1",
         "uuid": "^9.0.1"
     },
@@ -48,20 +69,5 @@
             "extensions": "ts",
             "quiet": false
         }
-    },
-    "devDependencies": {
-        "@types/node": "^18.7.23",
-        "nodemon": "^3.1.2",
-        "npm-watch": "^0.13.0",
-        "typescript": "^4.9.5"
-    },
-    "author": {
-        "name": "Needle",
-        "email": "help@needle.tools",
-        "url": "https://needle.tools/"
-    },
-    "repository": {
-        "type": "git",
-        "url": "git+https://github.com/needle-tools/needle-gltf-extensions.git"
     }
 }

package/dist/src/cache/cache.d.ts

@@ -48,6 +48,7 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
 /**
  * Get the hash for a property or an object.
  * @param property The property to get the hash for.
+ * @param level The current level of recursion.
  * @returns The hash as a number.
  */
 export declare function getHash(property: ExtensibleProperty | object, level?: number): number;

package/dist/src/cache/cache.js

@@ -167,6 +167,7 @@ export function getKeyWithHash(property) {
 /**
  * Get the hash for a property or an object.
  * @param property The property to get the hash for.
+ * @param level The current level of recursion.
  * @returns The hash as a number.
  */
 export function getHash(property, level = 0) {

package/dist/src/cli/index.js

@@ -7,8 +7,8 @@ import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache } from '../cache/cache.js';
 import { existsSync, statSync } from 'fs';
-import { foreachGLTF } from '../utils/fileutils.js';
-import { dirname } from 'path';
+import { foreachGLTF, getDirName } from '../utils/fileutils.js';
+import path, { dirname } from 'path';
 import { ERROR_CODES } from '../constants.js';
 // For testing / dev you can run `npm link` in the package directory
 // For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
@@ -30,10 +30,12 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
 Compress a single glTF file. The input file is compressed and written to the output file.  
 `)
     .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
-    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}] e.g. \`--usecase ${UsecaseOptions[0]}\"`, { validator: program.STRING })
+    .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
+    .option("--usecase <usecase>", `The usecase for the compression allows you to determine how your asset is supposed to be used:\nThis will set compression and transform settings on a high level if no other settings are found.\nPossible options: [${UsecaseOptions.join(", ")}] e.g. \"--usecase ${UsecaseOptions[0]}\"`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
-    .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
-    .option("--output-stats", "The path to a JSON file where the stats will be written to.", { validator: program.STRING })
+    .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
+    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
+    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
     .action(async ({ logger, args, options }) => {
     logger.info("Transforming glTF file: " + args.input);
     const stats = {
@@ -43,19 +45,21 @@ Compress a single glTF file. The input file is compressed and written to the out
         totalFileSizeInMB: 0,
         totalFileSizeInMBBefore: 0,
     };
-    let config = getConfig(options.config?.toString());
+    const file = args.input.toString();
+    let config = getConfig(options.config?.toString(), file);
     if (options.usecase) {
         logger.info("Using usecase: " + options.usecase);
         config = { ...createConfig(options.usecase.toString()), ...config, };
-        if (config.usecase != "unknown" && config.usecase != options.usecase.toString()) {
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
             logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
         }
         else {
             config.usecase = options.usecase.toString();
         }
     }
-    const file = args.input.toString();
-    const outDir = statSync(file).isDirectory() ? file : dirname(file);
+    const debug = options.debug === true;
+    const isDirectory = statSync(file).isDirectory();
+    const outDir = isDirectory ? file : dirname(file);
     let outputIsDirectory = false;
     if (statSync(file).isDirectory() && options.output) {
         if (statSync(options.output.toString()).isDirectory()) {
@@ -73,17 +77,20 @@ Compress a single glTF file. The input file is compressed and written to the out
     const file_stats = [];
     const set = new Set();
     await foreachGLTF(file, async (file) => {
-        let outputFile = options.output?.toString() || file;
+        let outputFile = args.output?.toString();
+        if (!outputFile?.length) {
+            outputFile = file;
+        }
         if (outputIsDirectory) {
             const filename = file.replaceAll("\\", "/").split("/").pop();
-            outputFile = options.output?.toString() + "/" + filename;
+            outputFile = args.output?.toString() + "/" + filename;
         }
         const res = await packGLTF(file, outputFile, {
             config,
             packedFiles: set,
             stats,
-            debug: false,
-            useCache: true,
+            debug,
+            useCache: options.cache !== false,
             logger: logger,
             file_stats: file_stats,
         });
@@ -91,10 +98,11 @@ Compress a single glTF file. The input file is compressed and written to the out
             process.exit(ERROR_CODES.PACKING_FAILED);
         }
     });
-    printStats(stats);
+    printStats(stats, logger, config);
     console.log("Stats: ", options);
-    if (options.outputStats) {
-        writeFileStatsToFile(file_stats, options.outputStats.toString());
+    if (options.stats) {
+        const statsFile = (isDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
+        writeFileStatsToFile(config, file_stats, statsFile);
     }
 })
     .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
@@ -103,10 +111,12 @@ This will produce multiple versions of the input file, each with a different lev
 Each version will be compressed and written to the output directory.
 `)
     .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
+    // .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
     .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
-    // .option("--output <output>", "The output directory. If not provided, the input files will be overwritten.", { validator: program.STRING })
-    .option("--output-stats", "The path to a JSON file where the stats will be written to.", { validator: program.STRING })
+    .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
+    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
+    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
     .action(async ({ logger, args, options }) => {
     logger.info("Transforming glTF file: " + args.input);
     const stats = {
@@ -116,30 +126,33 @@ Each version will be compressed and written to the output directory.
         totalFileSizeInMB: 0,
         totalFileSizeInMBBefore: 0,
     };
-    let config = getConfig(options.config?.toString());
+    const file = args.input.toString();
+    let config = getConfig(options.config?.toString(), file);
     if (options.usecase) {
         logger.info("Using usecase: " + options.usecase);
         config = { ...createConfig(options.usecase.toString()), ...config, };
-        if (config.usecase != "unknown" && config.usecase != options.usecase.toString()) {
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
             logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
         }
         else {
             config.usecase = options.usecase.toString();
         }
     }
-    const file = args.input.toString();
-    const outDir = statSync(file).isDirectory() ? file : dirname(file);
+    const isDirectory = statSync(file).isDirectory();
+    const outDir = isDirectory ? file : dirname(file);
     const file_stats = [];
     const set = new Set();
+    const useCache = options.cache !== false;
+    const debug = options.debug === true;
     await foreachGLTF(file, async (file) => {
-        logger.info("> Transform " + file);
+        logger.info("→ Transform " + file);
         const progressive_results = new Array();
         await make_progressive({
             path: file,
             results: progressive_results,
             config,
-            debug: false,
-            useCache: true,
+            debug,
+            useCache,
         }).catch(err => {
             logger.error(err);
             process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
@@ -148,25 +161,24 @@ Each version will be compressed and written to the output directory.
             config,
             stats,
             packedFiles: set,
-            debug: false,
-            useCache: true,
+            debug,
+            useCache,
             logger: logger,
             file_stats,
         };
         for (const res of progressive_results) {
-            logger.info("> Compress " + res);
             if (await packGLTF(res, res, opts) === false) {
                 process.exit(ERROR_CODES.PACKING_FAILED);
             }
         }
-        logger.info("> Compress " + file);
         if (await packGLTF(file, file, opts) === false) {
             process.exit(ERROR_CODES.PACKING_FAILED);
         }
     });
-    printStats(stats);
-    if (options.outputStats) {
-        writeFileStatsToFile(file_stats, options.outputStats.toString());
+    printStats(stats, logger, config);
+    if (options.stats) {
+        const statsFile = (isDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
+        writeFileStatsToFile(config, file_stats, statsFile);
     }
 });
 program.run();

package/dist/src/config/index.d.ts

@@ -1,4 +1,4 @@
-declare type Usecase = "unknown" | "product" | "world";
+declare type Usecase = "default" | "product" | "world";
 export declare const UsecaseOptions: string[];
 /**
  * The config object contains information about how a file should be processed.
@@ -6,6 +6,9 @@ export declare const UsecaseOptions: string[];
 export type Config = {
     /** High-level configuration */
     usecase: Usecase;
+    textures?: {
+        lods: boolean;
+    };
 };
 /**
  * Create a config object with the given usecase.
@@ -39,5 +42,6 @@ export declare function createConfig(usecase: string): {
  *
  *
  */
-export declare function getConfig(path?: string): Config;
+export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string): Config;
+export declare function searchConfig(directory: string): Config | null;
 export {};

package/dist/src/config/index.js

@@ -1,7 +1,8 @@
-import { existsSync, readFileSync } from "fs";
+import { existsSync, readFileSync, statSync } from "fs";
+import path, { dirname } from "path";
 export const UsecaseOptions = ["product"];
 const defaultConfig = {
-    usecase: "unknown",
+    usecase: "default",
 };
 /**
  * Create a config object with the given usecase.
@@ -40,49 +41,46 @@ export function createConfig(usecase) {
  *
  *
  */
-export function getConfig(path) {
-    if (path != null) {
-        if (!existsSync(path)) {
-            throw new Error(`No config found at \"${path}\"`);
+export function getConfig(configPath, fallbackSearchDirectory) {
+    if (configPath != null) {
+        if (!existsSync(configPath)) {
+            throw new Error(`No config found at \"${configPath}\"`);
+        }
+        const res = resolveConfig(configPath, readFileSync(configPath, "utf8"));
+        if (res) {
+            return res;
+        }
+        else {
+            throw new Error(`Failed parsing config at \"${configPath}\"`);
         }
-        return resolveConfig(path, readFileSync(path, "utf8"));
     }
     if (_cachedConfig !== undefined) {
         return _cachedConfig;
     }
     // Resolve the config once and cache it
-    const result = internalResolveConfig();
-    if (result) {
-        console.log("INFO: Using optimization config with usecase: " + result.usecase);
-        _cachedConfig = result;
+    // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
+    if (fallbackSearchDirectory) {
+        const result = searchConfig(fallbackSearchDirectory);
+        if (result) {
+            _cachedConfig = { ...defaultConfig, ...result };
+            console.log("INFO: Using config with usecase: " + _cachedConfig.usecase);
+        }
     }
-    else {
-        console.log("INFO: No optimization config found. Using default config.");
+    // If no config was found, use the default config
+    if (!_cachedConfig) {
+        console.log("INFO: No config found. Using default config.");
         _cachedConfig = defaultConfig;
     }
     return _cachedConfig;
 }
 let _cachedConfig = undefined;
-function internalResolveConfig() {
-    // Check if a config path is passed as an argument
-    const args = process.argv.slice(2);
-    for (let i = 0; i < args.length; i++) {
-        if (args[i] === "--config") {
-            if (i + 1 < args.length) {
-                const configPath = args[i + 1];
-                if (existsSync(configPath)) {
-                    const content = readFileSync(configPath, "utf8");
-                    return resolveConfig(configPath, content);
-                }
-                throw new Error(`Config file not found at path configured by CLI --config argument: \"${configPath}\"`);
-            }
-        }
+export function searchConfig(directory) {
+    if (statSync(directory).isFile()) {
+        directory = dirname(directory);
     }
-    // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
-    const workingDir = process.cwd();
-    let directory = workingDir.replaceAll("\\", "/");
+    directory = directory.replaceAll("\\", "/");
     while (directory.length > 0) {
-        const configPath = `${directory}/needle.config.json`;
+        const configPath = path.join(directory, "needle.config.json");
         if (existsSync(configPath)) {
             const content = readFileSync(configPath, "utf8");
             const config = resolveConfig(configPath, content);
@@ -103,16 +101,16 @@ function resolveConfig(filename, content) {
     if (filename.endsWith("needle.config.json")) {
         try {
             const config = JSON.parse(content);
-            if (config.optimization) {
-                console.log("INFO: Found optimization config at \"" + filename + "\".");
+            if (config.gltf) {
+                console.log("INFO: Found \"gltf\" config at \"" + filename + "\".");
                 // If the optimization config is a string, it's the usecase
-                if (typeof config.optimization === "string") {
-                    return createConfig(config.optimization);
+                if (typeof config.gltf === "string") {
+                    return createConfig(config.gltf);
                 }
-                return config.optimization;
+                return config.gltf;
             }
             else {
-                console.log("INFO: No optimization config found in needle.config.json. Using default config.");
+                console.log("INFO: No \"gltf\" config found in needle.config.json. Using default config.");
                 return null;
             }
         }

package/dist/src/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts

@@ -15,7 +15,7 @@ export interface IOpaqueProperty {
     /** register a reference so gltf-transform will not remove it in e.g. dedup
      * https://github.com/donmccurdy/glTF-Transform/issues/980#issuecomment-1585748202
      */
-    setReference(name: string, value: any): any;
+    setReference(name: string, value: any, attributes?: Record<string, any>): void;
 }
 export interface IExtensionWriter {
     /** used by progressive texture extension to pass an extension between documents */