@needle-tools/gltf-build-pipeline 2.1.0-alpha.6 → 2.2.0-alpha.10

package/CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.2.0-alpha.10] - 2023-06-13
+- fix: progressive transform of converted GLB files where meshes don't have indicies
+- fix: skip progressive texture generation for textures for textures that already match the lowest LOD size
+- fix: if compression is set to None the LOD textures should respect that setting and also not be compressed
+- fix: fallback mesh density calculation for meshes that aren't indexed
+- fix: NEEDLE_mesh_compression extension falsely overriding primitive extensions
+
+## [2.2.0-alpha.5] - 2023-06-06
+- add: CLI `--progressive` option to `transform` command (default is true)
+- add: CLI `--compress` option to `transform` command (default is true)
+- add: config `meshes: { lods: boolean }` option
+- remove: CLI "compress" command
+- change: enable texture LODs by default
+- fix: `--config` path can now point to a directory or a `needle.config.json` file as well
+- fix: mesh LOD regression caused by change in gltf-transform `weld` behaviour 
+- fix: KTX SOFTWARE >= 4.3.0 is installed
+
 ## [2.1.0-alpha.6] - 2023-06-04
 - fix: errors when invoking npm scripts caused by the cli program
 - fix: remove cache directory error

package/dist/{src/cache → cache}/cache.js

@@ -19,6 +19,7 @@ import { getVersion } from "../utils/version.js";
 //     return 0;
 // }
 const buildPipelineVersion = getVersion(false);
+const debugLogs = process.argv.includes("--debug");
 /**
  * Clear the cache.
  * This will remove all files in the cache directory.
@@ -86,7 +87,8 @@ export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
 export function addToCache(key, bytes) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
-    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
+    if (debugLogs)
+        console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
     fs.writeFileSync(path, bytes);
 }
 /**

package/dist/cli/index.js

@@ -0,0 +1,127 @@
+#! /usr/bin/env node
+import { program } from '@donmccurdy/caporal';
+import { packGLTF } from "../scripts/index.js";
+import { printStats, writeFileStatsToFile } from '../utils/stats.js';
+import { getVersion } from '../utils/version.js';
+import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
+import { make_progressive } from '../transforms/needle_progressive.js';
+import { cacheSizeLimit, clearCache } from '../cache/cache.js';
+import { statSync } from 'fs';
+import { foreachGLTF, getDirName } from '../utils/fileutils.js';
+import path, { dirname } from 'path';
+import { ERROR_CODES } from '../constants.js';
+// For testing / dev you can run `npm link` in the package directory
+// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
+program
+    .command("version", "Print current verison")
+    .action(async ({ logger }) => {
+    logger.info(getVersion());
+})
+    .command("clear-caches", "Clear all caches")
+    .help(`Clears all cached files
+Caches are limited to ${cacheSizeLimit} MB disc space by default.
+`)
+    .action(async ({ logger }) => {
+    logger.info("Clearing caches");
+    clearCache(logger);
+})
+    .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
+    .help(`
+This will produce multiple versions of the input file, each with a different level of optimization.   
+Each version will be compressed and written to the output directory.
+`)
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
+    // .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
+    .option("--compress", `When enabled files will be compressed`, { validator: program.BOOLEAN, default: true })
+    .option("--progressive", `When enabled files will processed to be progressively loaded`, { validator: program.BOOLEAN, default: true })
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
+    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
+    .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
+    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
+    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
+    .action(async ({ logger, args, options }) => {
+    if (options.compress === false && options.progressive === false) {
+        logger.error("You need to enable at least one of the options: --compress or --progressive");
+        return;
+    }
+    logger.info("Transforming glTF file: " + args.input);
+    const stats = {
+        startTime: Date.now(),
+        totalGPUMemoryInMB: 0,
+        totalFilesProcessed: 0,
+        totalFileSizeInMB: 0,
+        totalFileSizeInMBBefore: 0,
+    };
+    const file = args.input.toString();
+    let config = getConfig(options.config?.toString(), file, {
+        logger
+    });
+    if (options.usecase) {
+        logger.info("Using usecase: " + options.usecase);
+        config = { ...createConfig(options.usecase.toString()), ...config, };
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
+            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
+        }
+        else {
+            config.usecase = options.usecase.toString();
+        }
+    }
+    const output = options.output?.toString() || file;
+    const outputIsDirectory = statSync(output).isDirectory();
+    const outDir = outputIsDirectory ? output : dirname(output);
+    const file_stats = [];
+    const set = new Set();
+    const useCache = options.cache !== false;
+    const debug = options.debug === true;
+    await foreachGLTF(file, async (file) => {
+        logger.info("→ Transform " + file);
+        const progressive_results = new Array();
+        if (options.progressive === true) {
+            if (await make_progressive({
+                path: file,
+                results: progressive_results,
+                config,
+                debug,
+                useCache,
+            }).catch(err => {
+                logger.error(err);
+                process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
+            }) === false) {
+                process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
+            }
+        }
+        if (options.compress === true) {
+            const opts = {
+                config,
+                stats,
+                packedFiles: set,
+                debug,
+                useCache,
+                logger: logger,
+                file_stats,
+            };
+            for (const res of progressive_results) {
+                if (await packGLTF(res, res, opts) === false) {
+                    process.exit(ERROR_CODES.PACKING_FAILED);
+                }
+            }
+            if (await packGLTF(file, file, opts) === false) {
+                process.exit(ERROR_CODES.PACKING_FAILED);
+            }
+        }
+    });
+    printStats(stats, logger, config);
+    if (options.stats) {
+        const statsFile = (outputIsDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
+        writeFileStatsToFile(config, file_stats, statsFile);
+    }
+});
+program.run().catch(err => {
+    // Since we still invoke this package with paths when running npm scripts we need to ignore this error
+    // We might want to move this code here into an extra package.
+    if (err.message.includes("Unknown command") && !err.message.includes("Did you mean")) {
+        // ignore
+        return;
+    }
+    throw err;
+});

package/dist/{src/config → config}/index.d.ts

@@ -1,3 +1,4 @@
+import { ILogger } from "@gltf-transform/core";
 declare type Usecase = "default" | "product" | "world";
 export declare const UsecaseOptions: string[];
 /**
@@ -9,6 +10,9 @@ export type Config = {
     textures?: {
         lods: boolean;
     };
+    meshes?: {
+        lods: boolean;
+    };
 };
 /**
  * Create a config object with the given usecase.
@@ -42,6 +46,8 @@ export declare function createConfig(usecase: string): {
  *
  *
  */
-export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string): Config;
-export declare function searchConfig(directory: string): Config | null;
+export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string, opts?: {
+    logger: ILogger;
+}): Config;
+export declare function searchConfig(directory: string, logger: ILogger): Config | null;
 export {};

package/dist/{src/config → config}/index.js

@@ -1,5 +1,6 @@
+import { Logger, Verbosity } from "@gltf-transform/core";
 import { existsSync, readFileSync, statSync } from "fs";
-import path, { dirname } from "path";
+import path, { dirname, resolve } from "path";
 export const UsecaseOptions = ["product", "world"];
 const defaultConfig = {
     usecase: "default",
@@ -41,12 +42,27 @@ export function createConfig(usecase) {
  *
  *
  */
-export function getConfig(configPath, fallbackSearchDirectory) {
+export function getConfig(configPath, fallbackSearchDirectory, opts) {
+    let logger = opts?.logger;
+    if (!logger) {
+        logger = new Logger(Verbosity.WARN);
+    }
     if (configPath != null) {
+        configPath = resolve(process.cwd(), configPath);
         if (!existsSync(configPath)) {
             throw new Error(`No config found at \"${configPath}\"`);
         }
-        const res = resolveConfig(configPath, readFileSync(configPath, "utf8"));
+        const isDirectory = statSync(configPath).isDirectory();
+        if (isDirectory) {
+            const res = searchConfig(configPath, logger);
+            if (res) {
+                return res;
+            }
+            else {
+                throw new Error(`No config found in directory \"${configPath}\"`);
+            }
+        }
+        const res = resolveConfig(configPath, readFileSync(configPath, "utf8"), logger);
         if (res) {
             return res;
         }
@@ -60,21 +76,21 @@ export function getConfig(configPath, fallbackSearchDirectory) {
     // Resolve the config once and cache it
     // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
     if (fallbackSearchDirectory) {
-        const result = searchConfig(fallbackSearchDirectory);
+        const result = searchConfig(fallbackSearchDirectory, logger);
         if (result) {
             _cachedConfig = { ...defaultConfig, ...result };
-            console.log("INFO: Using config with usecase: " + _cachedConfig.usecase);
+            logger.info("Using config with usecase: " + _cachedConfig.usecase);
         }
     }
     // If no config was found, use the default config
     if (!_cachedConfig) {
-        console.log("INFO: No config found. Using default config.");
+        logger.info("No config found. Using default config.");
         _cachedConfig = defaultConfig;
     }
     return _cachedConfig;
 }
 let _cachedConfig = undefined;
-export function searchConfig(directory) {
+export function searchConfig(directory, logger) {
     if (statSync(directory).isFile()) {
         directory = dirname(directory);
     }
@@ -83,7 +99,7 @@ export function searchConfig(directory) {
         const configPath = path.join(directory, "needle.config.json");
         if (existsSync(configPath)) {
             const content = readFileSync(configPath, "utf8");
-            const config = resolveConfig(configPath, content);
+            const config = resolveConfig(configPath, content, logger);
             if (config) {
                 return config;
             }
@@ -96,13 +112,13 @@ export function searchConfig(directory) {
     }
     return null;
 }
-function resolveConfig(filename, content) {
+function resolveConfig(filename, content, logger) {
     // Handle config being part of `needle.config.json`
     if (filename.endsWith("needle.config.json")) {
         try {
             const config = JSON.parse(content);
             if (config.gltf) {
-                console.log("INFO: Found \"gltf\" config at \"" + filename + "\".");
+                logger.info("Found \"gltf\" config at \"" + filename + "\".");
                 // If the optimization config is a string, it's the usecase
                 if (typeof config.gltf === "string") {
                     return createConfig(config.gltf);
@@ -110,12 +126,12 @@ function resolveConfig(filename, content) {
                 return config.gltf;
             }
             else {
-                console.log("INFO: No \"gltf\" config found in needle.config.json. Using default config.");
+                logger.info("No \"gltf\" config found in needle.config.json. Using default config.");
                 return null;
             }
         }
         catch (err) {
-            console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
+            logger.error(`Failed parsing config \"${err.message}\" at ${filename}`);
         }
     }
     // Handle config being the content of the file
@@ -125,12 +141,12 @@ function resolveConfig(filename, content) {
         }
         catch (err) {
             if (err instanceof Error)
-                console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
+                logger.error(`Failed parsing config \"${err.message}\" at ${filename}`);
             else
-                console.error(`ERR: Failed parsing config \"${err}\" at ${filename}`);
+                logger.error(`Failed parsing config \"${err}\" at ${filename}`);
         }
     }
     else {
-        console.error("ERR: Invalid config file format. Expected JSON object at " + filename);
+        logger.error("Invalid config file format. Expected JSON object at " + filename);
     }
 }

package/dist/{src/extensions → extensions}/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js

@@ -62,7 +62,7 @@ export class NEEDLE_mesh_compression extends Extension {
                 if (ext) {
                     const prim = meshDef.primitives[i];
                     if (prim) {
-                        prim.extensions = meshDef.extensions || {};
+                        prim.extensions = prim.extensions || {};
                         prim.extensions[EXTENSION_NAME] = ext.model;
                     }
                 }

package/dist/functions/weld.3.d.ts

@@ -0,0 +1,76 @@
+import { Document, Primitive, Transform } from '@gltf-transform/core';
+/** @hidden */
+export declare function createIndices(count: number, maxIndex?: number): Uint16Array | Uint32Array;
+export declare function formatLong(x: number): string;
+export declare function formatDelta(a: number, b: number, decimals?: number): string;
+export declare function formatDeltaOp(a: number, b: number): string;
+export declare function cleanPrimitive(prim: Primitive): void;
+/** Options for the {@link weld} function. */
+export interface WeldOptions {
+    /** Tolerance for vertex positions, as a fraction of primitive AABB. */
+    tolerance?: number;
+    /** Tolerance for vertex normals, in radians. */
+    toleranceNormal?: number;
+    /** Whether to overwrite existing indices. */
+    overwrite?: boolean;
+    /** Enables a more thorough, but slower, search for vertices to weld. */
+    exhaustive?: boolean;
+}
+export declare const WELD_DEFAULTS: Required<WeldOptions>;
+/**
+ * Index {@link Primitive Primitives} and (optionally) merge similar vertices. When merged
+ * and indexed, data is shared more efficiently between vertices. File size can
+ * be reduced, and the GPU can sometimes use the vertex cache more efficiently.
+ *
+ * When welding, the 'tolerance' threshold determines which vertices qualify for
+ * welding based on distance between the vertices as a fraction of the primitive's
+ * bounding box (AABB). For example, tolerance=0.01 welds vertices within +/-1%
+ * of the AABB's longest dimension. Other vertex attributes are also compared
+ * during welding, with attribute-specific thresholds. For `tolerance=0`, geometry
+ * is indexed in place, without merging.
+ *
+ * To preserve visual appearance consistently, use low `toleranceNormal` thresholds
+ * around 0.1 (±3º). To pre-processing a scene before simplification or LOD creation,
+ * use higher thresholds around 0.5 (±30º).
+ *
+ * Example:
+ *
+ * ```javascript
+ * import { weld } from '@gltf-transform/functions';
+ *
+ * await document.transform(
+ * 	weld({ tolerance: 0.001, toleranceNormal: 0.5 })
+ * );
+ * ```
+ *
+ * @category Transforms
+ */
+export declare function weld(_options?: WeldOptions): Transform;
+/**
+ * Index a {@link Primitive} and (optionally) weld similar vertices. When merged
+ * and indexed, data is shared more efficiently between vertices. File size can
+ * be reduced, and the GPU can sometimes use the vertex cache more efficiently.
+ *
+ * When welding, the 'tolerance' threshold determines which vertices qualify for
+ * welding based on distance between the vertices as a fraction of the primitive's
+ * bounding box (AABB). For example, tolerance=0.01 welds vertices within +/-1%
+ * of the AABB's longest dimension. Other vertex attributes are also compared
+ * during welding, with attribute-specific thresholds. For tolerance=0, geometry
+ * is indexed in place, without merging.
+ *
+ * Example:
+ *
+ * ```javascript
+ * import { weldPrimitive } from '@gltf-transform/functions';
+ *
+ * const mesh = document.getRoot().listMeshes()
+ * 	.find((mesh) => mesh.getName() === 'Gizmo');
+ *
+ * for (const prim of mesh.listPrimitives()) {
+ *   weldPrimitive(prim, {tolerance: 0.0001});
+ * }
+ * ```
+ *
+ * @privateRemarks TODO(v4): Remove the "Document" parameter.
+ */
+export declare function weldPrimitive(a: Document | Primitive, b?: Primitive | WeldOptions, c?: Required<WeldOptions>): void;