@needle-tools/gltf-build-pipeline 2.1.0-alpha.6 → 2.2.0-alpha.1

package/CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.2.0-alpha] - 2023-06-06
+- Add: CLI `--progressive` option to `transform` command (default is enabled)
+- Add: config `meshes: { lods: boolean }` option
+- Remove: CLI "compress" command
+- Change: enable texture LODs by default
+- Fix: `--config` path can now point to a directory or a `needle.config.json` file as well
+
 ## [2.1.0-alpha.6] - 2023-06-04
 - fix: errors when invoking npm scripts caused by the cli program
 - fix: remove cache directory error

package/dist/cli/index.js

@@ -0,0 +1,118 @@
+#! /usr/bin/env node
+import { program } from '@donmccurdy/caporal';
+import { packGLTF } from "../scripts/index.js";
+import { printStats, writeFileStatsToFile } from '../utils/stats.js';
+import { getVersion } from '../utils/version.js';
+import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
+import { make_progressive } from '../transforms/needle_progressive.js';
+import { cacheSizeLimit, clearCache } from '../cache/cache.js';
+import { statSync } from 'fs';
+import { foreachGLTF, getDirName } from '../utils/fileutils.js';
+import path, { dirname } from 'path';
+import { ERROR_CODES } from '../constants.js';
+// For testing / dev you can run `npm link` in the package directory
+// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
+program
+    .command("version", "Print current verison")
+    .action(async ({ logger }) => {
+    logger.info(getVersion());
+})
+    .command("clear-caches", "Clear all caches")
+    .help(`Clears all cached files
+Caches are limited to ${cacheSizeLimit} MB disc space by default.
+`)
+    .action(async ({ logger }) => {
+    logger.info("Clearing caches");
+    clearCache(logger);
+})
+    .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
+    .help(`
+This will produce multiple versions of the input file, each with a different level of optimization.   
+Each version will be compressed and written to the output directory.
+`)
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
+    // .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
+    .option("--progressive", `When enabled files will processed to be progressively loaded`, { validator: program.BOOLEAN, default: true })
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
+    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
+    .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
+    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
+    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
+    .action(async ({ logger, args, options }) => {
+    logger.info("Transforming glTF file: " + args.input);
+    const stats = {
+        startTime: Date.now(),
+        totalGPUMemoryInMB: 0,
+        totalFilesProcessed: 0,
+        totalFileSizeInMB: 0,
+        totalFileSizeInMBBefore: 0,
+    };
+    const file = args.input.toString();
+    let config = getConfig(options.config?.toString(), file, {
+        logger
+    });
+    if (options.usecase) {
+        logger.info("Using usecase: " + options.usecase);
+        config = { ...createConfig(options.usecase.toString()), ...config, };
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
+            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
+        }
+        else {
+            config.usecase = options.usecase.toString();
+        }
+    }
+    const output = options.output?.toString() || file;
+    const outputIsDirectory = statSync(output).isDirectory();
+    const outDir = outputIsDirectory ? output : dirname(output);
+    const file_stats = [];
+    const set = new Set();
+    const useCache = options.cache !== false;
+    const debug = options.debug === true;
+    await foreachGLTF(file, async (file) => {
+        logger.info("→ Transform " + file);
+        const progressive_results = new Array();
+        if (options.progressive === true) {
+            await make_progressive({
+                path: file,
+                results: progressive_results,
+                config,
+                debug,
+                useCache,
+            }).catch(err => {
+                logger.error(err);
+                process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
+            });
+        }
+        const opts = {
+            config,
+            stats,
+            packedFiles: set,
+            debug,
+            useCache,
+            logger: logger,
+            file_stats,
+        };
+        for (const res of progressive_results) {
+            if (await packGLTF(res, res, opts) === false) {
+                process.exit(ERROR_CODES.PACKING_FAILED);
+            }
+        }
+        if (await packGLTF(file, file, opts) === false) {
+            process.exit(ERROR_CODES.PACKING_FAILED);
+        }
+    });
+    printStats(stats, logger, config);
+    if (options.stats) {
+        const statsFile = (outputIsDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
+        writeFileStatsToFile(config, file_stats, statsFile);
+    }
+});
+program.run().catch(err => {
+    // Since we still invoke this package with paths when running npm scripts we need to ignore this error
+    // We might want to move this code here into an extra package.
+    if (err.message.includes("Unknown command") && !err.message.includes("Did you mean")) {
+        // ignore
+        return;
+    }
+    throw err;
+});

package/dist/{src/config → config}/index.d.ts

@@ -1,3 +1,4 @@
+import { ILogger } from "@gltf-transform/core";
 declare type Usecase = "default" | "product" | "world";
 export declare const UsecaseOptions: string[];
 /**
@@ -9,6 +10,9 @@ export type Config = {
     textures?: {
         lods: boolean;
     };
+    meshes?: {
+        lods: boolean;
+    };
 };
 /**
  * Create a config object with the given usecase.
@@ -42,6 +46,8 @@ export declare function createConfig(usecase: string): {
  *
  *
  */
-export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string): Config;
-export declare function searchConfig(directory: string): Config | null;
+export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string, opts?: {
+    logger: ILogger;
+}): Config;
+export declare function searchConfig(directory: string, logger: ILogger): Config | null;
 export {};

package/dist/{src/config → config}/index.js

@@ -1,5 +1,6 @@
+import { Logger, Verbosity } from "@gltf-transform/core";
 import { existsSync, readFileSync, statSync } from "fs";
-import path, { dirname } from "path";
+import path, { dirname, resolve } from "path";
 export const UsecaseOptions = ["product", "world"];
 const defaultConfig = {
     usecase: "default",
@@ -41,12 +42,27 @@ export function createConfig(usecase) {
  *
  *
  */
-export function getConfig(configPath, fallbackSearchDirectory) {
+export function getConfig(configPath, fallbackSearchDirectory, opts) {
+    let logger = opts?.logger;
+    if (!logger) {
+        logger = new Logger(Verbosity.WARN);
+    }
     if (configPath != null) {
+        configPath = resolve(process.cwd(), configPath);
         if (!existsSync(configPath)) {
             throw new Error(`No config found at \"${configPath}\"`);
         }
-        const res = resolveConfig(configPath, readFileSync(configPath, "utf8"));
+        const isDirectory = statSync(configPath).isDirectory();
+        if (isDirectory) {
+            const res = searchConfig(configPath, logger);
+            if (res) {
+                return res;
+            }
+            else {
+                throw new Error(`No config found in directory \"${configPath}\"`);
+            }
+        }
+        const res = resolveConfig(configPath, readFileSync(configPath, "utf8"), logger);
         if (res) {
             return res;
         }
@@ -60,21 +76,21 @@ export function getConfig(configPath, fallbackSearchDirectory) {
     // Resolve the config once and cache it
     // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
     if (fallbackSearchDirectory) {
-        const result = searchConfig(fallbackSearchDirectory);
+        const result = searchConfig(fallbackSearchDirectory, logger);
         if (result) {
             _cachedConfig = { ...defaultConfig, ...result };
-            console.log("INFO: Using config with usecase: " + _cachedConfig.usecase);
+            logger.info("Using config with usecase: " + _cachedConfig.usecase);
         }
     }
     // If no config was found, use the default config
     if (!_cachedConfig) {
-        console.log("INFO: No config found. Using default config.");
+        logger.info("No config found. Using default config.");
         _cachedConfig = defaultConfig;
     }
     return _cachedConfig;
 }
 let _cachedConfig = undefined;
-export function searchConfig(directory) {
+export function searchConfig(directory, logger) {
     if (statSync(directory).isFile()) {
         directory = dirname(directory);
     }
@@ -83,7 +99,7 @@ export function searchConfig(directory) {
         const configPath = path.join(directory, "needle.config.json");
         if (existsSync(configPath)) {
             const content = readFileSync(configPath, "utf8");
-            const config = resolveConfig(configPath, content);
+            const config = resolveConfig(configPath, content, logger);
             if (config) {
                 return config;
             }
@@ -96,13 +112,13 @@ export function searchConfig(directory) {
     }
     return null;
 }
-function resolveConfig(filename, content) {
+function resolveConfig(filename, content, logger) {
     // Handle config being part of `needle.config.json`
     if (filename.endsWith("needle.config.json")) {
         try {
             const config = JSON.parse(content);
             if (config.gltf) {
-                console.log("INFO: Found \"gltf\" config at \"" + filename + "\".");
+                logger.info("Found \"gltf\" config at \"" + filename + "\".");
                 // If the optimization config is a string, it's the usecase
                 if (typeof config.gltf === "string") {
                     return createConfig(config.gltf);
@@ -110,12 +126,12 @@ function resolveConfig(filename, content) {
                 return config.gltf;
             }
             else {
-                console.log("INFO: No \"gltf\" config found in needle.config.json. Using default config.");
+                logger.info("No \"gltf\" config found in needle.config.json. Using default config.");
                 return null;
             }
         }
         catch (err) {
-            console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
+            logger.error(`Failed parsing config \"${err.message}\" at ${filename}`);
         }
     }
     // Handle config being the content of the file
@@ -125,12 +141,12 @@ function resolveConfig(filename, content) {
         }
         catch (err) {
             if (err instanceof Error)
-                console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
+                logger.error(`Failed parsing config \"${err.message}\" at ${filename}`);
             else
-                console.error(`ERR: Failed parsing config \"${err}\" at ${filename}`);
+                logger.error(`Failed parsing config \"${err}\" at ${filename}`);
         }
     }
     else {
-        console.error("ERR: Invalid config file format. Expected JSON object at " + filename);
+        logger.error("Invalid config file format. Expected JSON object at " + filename);
     }
 }

package/dist/{src/transforms → transforms}/needle_mesh_transform.js

@@ -37,7 +37,7 @@ async function applyMeshCompression(document, transform_context, context) {
     //     return transform(document, transform_context);
     // }
     // by default we still want to run draco if the root doesnt have a compression model configured
-    console.log(`[${NAME}]: → will apply draco mesh compression (default)`);
+    document.getLogger().info(`[${NAME}]: → will apply draco mesh compression (default)`);
     return draco()(document, transform_context);
 }
 async function simplifierTransform(document, _context, _options) {

package/dist/{src/transforms → transforms}/needle_progressive.js

@@ -13,7 +13,6 @@ import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive } from './needle_mesh_transform.js';
 import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
 export async function make_progressive(opts) {
-    console.log(`INFO: Run progressive transform: ${opts.path}`);
     const file = opts.path;
     return foreachGLTF(file, async (f) => {
         // Don't re-process files that are already LODs
@@ -64,15 +63,17 @@ async function onProcess(file, opts) {
             }
             const transforms = [
                 needle_asset(),
-                make_progressive_textures(file, { size: [smallesTextureSize, smallesTextureSize] }, opts),
-                sparse(),
             ];
+            if (opts.config?.textures?.lods !== false) {
+                transforms.push(make_progressive_textures(file, { size: [smallesTextureSize, smallesTextureSize] }, opts));
+            }
             // TODO: progressive meshes aren't supported yet for VRMs
-            if (!file.endsWith(".vrm")) {
+            if (!file.endsWith(".vrm") && opts.config?.meshes?.lods !== false) {
                 transforms.push(make_progressive_meshes(file, {}, opts));
             }
             else
                 logger.warn("WARN: Progressive meshes are not supported for VRM files yet");
+            transforms.push(sparse());
             logger.info("→ Transforming: " + file);
             await document.transform(...transforms);
             let filename = file;
@@ -174,11 +175,11 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             const currentSize = Math.min(...texture.getSize());
             const LOD_LEVEL = new Array();
             LOD_LEVEL.push({ size: currentSize });
-            let generateLods = opts.config?.usecase === "world";
+            let generateLods = true; //opts.config?.usecase === "world";
             // If lods are enabled in the config
-            generateLods ||= opts.config?.textures?.lods === true;
-            // Ensure that texture lods are not disabled by { textures: { lods: false } }
-            generateLods &&= opts.config?.textures?.lods !== false;
+            // generateLods ||= opts.config?.textures?.lods === true;
+            // // Ensure that texture lods are not disabled by { textures: { lods: false } }
+            // generateLods &&= opts.config?.textures?.lods !== false;
             if (generateLods) {
                 const LEVEL = 6;
                 let lastSize = currentSize;
@@ -619,7 +620,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
             const lodLevelInFile = generatedLODInfos.length;
             const clonedData = { primitives: [], weights: [...mesh.getWeights()] };
             for (const prim of mesh.listPrimitives()) {
-                const data = clonePrimitiveData(prim);
+                const data = clonePrimitiveData(prim, logger);
                 if (data != null) {
                     clonedData.primitives.push(data);
                 }
@@ -689,7 +690,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
         }
     });
 }
-function clonePrimitiveData(prim) {
+function clonePrimitiveData(prim, logger) {
     const semantics = prim.listSemantics();
     if (semantics.some(s => s.includes("JOINTS"))) {
         return null;
@@ -697,7 +698,7 @@ function clonePrimitiveData(prim) {
     const indices = new Uint32Array(prim.getIndices().getArray());
     const attributes = {};
     for (const attr of semantics) {
-        console.log(`[${NAME}] Clone attribute ${attr}`);
+        logger.info(`[${NAME}] Clone attribute ${attr}`);
         const accessor = prim.getAttribute(attr);
         if (accessor) {
             attributes[attr] = new Float32Array(accessor.getArray());
@@ -725,6 +726,7 @@ function setPrimitiveData(prim, data) {
  * @param error the error to use for simplification. Higher means more simplification
  */
 async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLevel, opts) {
+    const logger = doc.getLogger();
     const fileExtension = ".glb";
     const fileName_withoutExtension = "mesh_lod_" + level + "_" + guid;
     const fileName = fileName_withoutExtension + fileExtension;
@@ -744,7 +746,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
         const cachedValue = tryGetFromCache(cacheKey);
         const meta = tryGetMetaFromCache(cacheKey);
         if (cachedValue && meta) {
-            console.log(`[${NAME}] LOD ${level} loaded ${(cachedValue.length / 1024).toFixed()} KB from cache for \"${mesh.getName()}\"`);
+            logger.info(`[${NAME}] LOD ${level} loaded ${(cachedValue.length / 1024).toFixed()} KB from cache for \"${mesh.getName()}\"`);
             writeFileSync(outPath, cachedValue);
             return { level, path: fileName, success: true, cacheKey: cacheKey, content_hash: hash, lodMeshInfo: meta.meshInfo, density: meta.density };
         }
@@ -783,7 +785,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
                 newPrim.setAttribute(semantics[i], newAttr);
             }
             else {
-                console.warn(`[${NAME}] Missing attribute ${semantics[i]} in mesh LOD ${level}`);
+                logger.warn(`[${NAME}] Missing attribute ${semantics[i]} in mesh LOD ${level}`);
             }
         }
         const indices = prim.getIndices();
@@ -818,7 +820,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
                 newPrim.setExtension(name, copy);
             }
             catch (err) {
-                console.error(`WARN: [${NAME}] Failed to copy extension ${ext.extensionName} for mesh LOD ${level} → ${err}`);
+                logger.error(`[${NAME}] Failed to copy extension ${ext.extensionName} for mesh LOD ${level} → ${err}`);
             }
         }
     }
@@ -836,7 +838,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     tasks.length = 0;
     const newMeshInformation = getMeshInformation(newMesh);
     if (newMeshInformation.totalVertexCount < 4) {
-        console.error(`[${NAME}] mesh LOD ${level} is has less than 4 vertices → skipping`);
+        logger.error(`[${NAME}] mesh LOD ${level} is has less than 4 vertices → skipping`);
         return { level, path: fileName, success: false, cacheKey: cacheKey, content_hash: hash, lodMeshInfo: null, density: null, };
     }
     let density = 0;
@@ -846,7 +848,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     }
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
     if (!progressiveExt || !progressiveExt.createMesh) {
-        console.error("Failed to create progressive extension");
+        logger.error("Failed to create progressive extension");
         return;
     }
     newMesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createMesh({
@@ -859,13 +861,13 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     if (compressionExt) {
         const newExt = newDoc.createExtension(NEEDLE_mesh_compression);
         const newProp = newExt.create(compressionExt.model);
-        console.log(`[${NAME}] Pass compression format to mesh LOD ${level} → ${JSON.stringify(compressionExt.model)}`);
+        logger.info(`[${NAME}] Pass compression format to mesh LOD ${level} → ${JSON.stringify(compressionExt.model)}`);
         newDoc.getRoot().setExtension(NEEDLE_mesh_compression.EXTENSION_NAME, newProp);
     }
     await newDoc.transform(sparse());
     const newIo = new NodeIO();
     registerExtensions(newIo);
-    console.log(`[${NAME}] Write mesh LOD ${level} with to \"${fileName}\"`);
+    logger.info(`[${NAME}] Write mesh LOD ${level} with to \"${fileName}\"`);
     await newIo.write(outPath, newDoc);
     // const hasChanged = meshInfo.maxVertexCount == newMeshInformation.maxVertexCount && meshInfo.totalVertexCount == newMeshInformation.totalVertexCount;
     // add to cache

package/dist/{src/transforms → transforms}/needle_toktx.js

@@ -1,10 +1,10 @@
 import { BufferUtils, FileUtils, ImageUtils, uuid } from "@gltf-transform/core";
 import { listTextureSlots, getTextureChannelMask, fitPowerOfTwo, TextureResizeFilter } from "@gltf-transform/functions";
-import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "@gltf-transform/cli";
 import { KHRTextureBasisu } from "@gltf-transform/extensions";
 import { checkKTXSoftware, createParams, Mode } from "./toktx.js";
 import { existsSync, mkdirSync } from "fs";
 import { formatBytes, spawn, waitExit } from './util.js';
+import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "./toktx_types.js";
 import fs from 'fs/promises';
 import os from 'os';
 import tmp from 'tmp';

package/dist/{src/transforms → transforms}/toktx.d.ts

@@ -1,5 +1,5 @@
-import { ETC1SOptions, UASTCOptions } from "@gltf-transform/cli";
 import { Document, ILogger, vec2, Texture } from '@gltf-transform/core';
+import { ETC1SOptions, UASTCOptions } from './toktx_types.js';
 export declare const Filter: {
     BOX: string;
     TENT: string;

package/dist/{src/transforms → transforms}/toktx.js

@@ -1,17 +1,13 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
 import micromatch from 'micromatch';
 import os from 'os';
 import semver from 'semver';
 import tmp from 'tmp';
-import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "@gltf-transform/cli";
 import { TextureChannel } from '@gltf-transform/core';
-// import { TextureBasisu } from '@gltf-transform/extensions';
-// import { getTextureChannelMask, listTextureSlots } from '@gltf-transform/functions';
 import { execSync } from 'child_process';
 import { spawn, commandExists, waitExit, MICROMATCH_OPTIONS } from './util.js';
 import { TextureResizeFilter, getTextureColorSpace } from '@gltf-transform/functions';
-// import { existsSync, mkdirSync } from 'fs';
-// import { tryGetFromCache, getKey, addToCache } from '../cache';
-// import { resizeTextureIfNecessary } from "./needle_texture_transform"
+import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from './toktx_types.js';
 tmp.setGracefulCleanup();
 const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
 const KTX_SOFTWARE_VERSION_MIN = '4.0.0-rc1';

package/dist/transforms/toktx_types.d.ts

@@ -0,0 +1,78 @@
+import { vec2 } from "@gltf-transform/core";
+import { TextureResizeFilter } from "@gltf-transform/functions";
+/**********************************************************************************************
+ * Interfaces.
+ */
+export declare const Mode: {
+    ETC1S: string;
+    UASTC: string;
+};
+export declare const Filter: {
+    BOX: string;
+    TENT: string;
+    BELL: string;
+    BSPLINE: string;
+    MITCHELL: string;
+    LANCZOS3: string;
+    LANCZOS4: string;
+    LANCZOS6: string;
+    LANCZOS12: string;
+    BLACKMAN: string;
+    KAISER: string;
+    GAUSSIAN: string;
+    CATMULLROM: string;
+    QUADRATIC_INTERP: string;
+    QUADRATIC_APPROX: string;
+    QUADRATIC_MIX: string;
+};
+interface GlobalOptions {
+    /** Instance of the Sharp encoder, required if resizing textures. */
+    encoder: unknown;
+    mode: string;
+    /** Pattern identifying textures to compress, matched to name or URI. */
+    pattern?: RegExp | null;
+    /**
+     * Pattern matching the material texture slot(s) to be compressed or converted.
+     * Passing a string (glob) is deprecated; use a RegExp instead.
+     */
+    slots?: RegExp | null;
+    /** Interpolation used for generating mipmaps. Default: 'lanczos4'. */
+    filter?: string;
+    filterScale?: number;
+    resize?: vec2 | 'nearest-pot' | 'ceil-pot' | 'floor-pot';
+    /** Interpolation used if resizing. Default: TextureResizeFilter.LANCZOS3. */
+    resizeFilter?: TextureResizeFilter;
+    jobs?: number;
+    /**
+     * Whether to clean up temporary files created during texture compression. See
+     * verbose log output for temporary file paths. Default: true.
+     */
+    cleanup?: boolean;
+    /**
+     * Attempts to avoid processing images that could exceed memory or other other
+     * limits, throwing an error instead. Default: true.
+     * @experimental
+     */
+    limitInputPixels?: boolean;
+}
+export interface ETC1SOptions extends GlobalOptions {
+    quality?: number;
+    compression?: number;
+    maxEndpoints?: number;
+    maxSelectors?: number;
+    rdo?: boolean;
+    rdoThreshold?: number;
+}
+export interface UASTCOptions extends GlobalOptions {
+    level?: number;
+    rdo?: boolean;
+    rdoLambda?: number;
+    rdoDictionarySize?: number;
+    rdoBlockScale?: number;
+    rdoStdDev?: number;
+    rdoMultithreading?: boolean;
+    zstd?: number;
+}
+export declare const ETC1S_DEFAULTS: Omit<ETC1SOptions, 'encoder' | 'mode'>;
+export declare const UASTC_DEFAULTS: Omit<UASTCOptions, 'encoder' | 'mode'>;
+export {};

package/dist/transforms/toktx_types.js

@@ -0,0 +1,62 @@
+import os from 'os';
+import { TextureChannel } from "@gltf-transform/core";
+import { TextureResizeFilter } from "@gltf-transform/functions";
+const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
+const KTX_SOFTWARE_VERSION_MIN = '4.3.0';
+const { R, G, A } = TextureChannel;
+/**********************************************************************************************
+ * Interfaces.
+ */
+export const Mode = {
+    ETC1S: 'etc1s',
+    UASTC: 'uastc',
+};
+export const Filter = {
+    BOX: 'box',
+    TENT: 'tent',
+    BELL: 'bell',
+    BSPLINE: 'b-spline',
+    MITCHELL: 'mitchell',
+    LANCZOS3: 'lanczos3',
+    LANCZOS4: 'lanczos4',
+    LANCZOS6: 'lanczos6',
+    LANCZOS12: 'lanczos12',
+    BLACKMAN: 'blackman',
+    KAISER: 'kaiser',
+    GAUSSIAN: 'gaussian',
+    CATMULLROM: 'catmullrom',
+    QUADRATIC_INTERP: 'quadratic_interp',
+    QUADRATIC_APPROX: 'quadratic_approx',
+    QUADRATIC_MIX: 'quadratic_mix',
+};
+const GLOBAL_DEFAULTS = {
+    resizeFilter: TextureResizeFilter.LANCZOS3,
+    filter: Filter.LANCZOS4,
+    filterScale: 1,
+    pattern: null,
+    slots: null,
+    // See: https://github.com/donmccurdy/glTF-Transform/pull/389#issuecomment-1089842185
+    jobs: 2 * NUM_CPUS,
+    cleanup: true,
+    limitInputPixels: true,
+};
+export const ETC1S_DEFAULTS = {
+    quality: 128,
+    compression: 1,
+    rdo: true,
+    rdoThreshold: 1.25,
+    maxSelectors: 0,
+    maxEndpoints: 0,
+    ...GLOBAL_DEFAULTS,
+};
+export const UASTC_DEFAULTS = {
+    level: 2,
+    rdo: false,
+    rdoLambda: 1.0,
+    rdoDictionarySize: 32768,
+    rdoBlockScale: 10.0,
+    rdoStdDev: 18.0,
+    rdoMultithreading: true,
+    zstd: 18,
+    ...GLOBAL_DEFAULTS,
+};

package/dist/{src/utils → utils}/version.js

@@ -1,4 +1,4 @@
-import packageJson from "../../package.json" assert { type: "json" };
+import packageJson from "#package.json" assert { type: "json" };
 export function getVersion(_marklocal = true) {
     let res = packageJson.version;
     return res;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.1.0-alpha.6",
+  "version": "2.2.0-alpha.1",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
-  "main": "index.mjs",
+  "main": "src/index.js",
   "type": "module",
   "readme": "README.md",
   "author": {
@@ -15,14 +15,17 @@
     "url": "git+https://github.com/needle-tools/needle-gltf-extensions.git"
   },
   "bin": {
-    "gltf-build-pipeline": "./dist/src/cli/index.js",
-    "needle-gltf": "./dist/src/cli/index.js"
+    "gltf-build-pipeline": "dist/cli/index.js",
+    "needle-gltf": "dist/cli/index.js"
+  },
+  "imports": {
+    "#package.json": "./package.json"
   },
   "scripts": {
     "start": "node scripts/index.mjs",
     "dev": "npm-watch compile",
-    "compile": "tsc || exit 1",
-    "clear-caches":  "node scripts/clear-caches.mjs",
+    "compile": "node tools/prebuild.mjs && (tsc || exit 1) && node tools/postbuild.mjs",
+    "clear-caches": "node scripts/clear-caches.mjs",
     "limit-caches": "node scripts/limit-caches.mjs",
     "transform": "node scripts/index.mjs",
     "transform:make-progressive": "node scripts/make-progressive.mjs",
@@ -31,7 +34,6 @@
     "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs"
   },
   "devDependencies": {
-    "@img/sharp-win32-x64": "^0.33.4",
     "@types/node": "^18.7.23",
     "npm-watch": "^0.13.0",
     "typescript": "^4.9.5"
@@ -44,7 +46,6 @@
   },
   "dependencies": {
     "@donmccurdy/caporal": "~0.0.10",
-    "@gltf-transform/cli": "^4.0.1",
     "@gltf-transform/core": "^4.0.1",
     "@gltf-transform/extensions": "^4.0.1",
     "@gltf-transform/functions": "^4.0.1",
@@ -72,4 +73,4 @@
       "quiet": false
     }
   }
-}
+}

package/tsconfig.json

@@ -1,16 +1,20 @@
 {
     "compilerOptions": {
         "target": "ESNext",
+        "rootDir": "src",
         "outDir": "dist",
         "declaration": true,
         "moduleResolution": "NodeNext",
         "module": "NodeNext",
         "esModuleInterop": true,
         "skipLibCheck": true,
-        "resolveJsonModule": true
-        
+        "resolveJsonModule": true,
+        "lib": [
+            "ESNext",
+            "DOM"
+        ],
     },
     "include": [
-        "src/index.ts"
+        "src/index.ts",
     ]
 }

package/dist/___config.json

@@ -1,3 +0,0 @@
-{
-    "usecase": "test"
-}

package/dist/package.json

@@ -1,75 +0,0 @@
-{
-    "name": "@needle-tools/gltf-build-pipeline",
-    "version": "2.1.0-alpha.6",
-    "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
-    "main": "index.mjs",
-    "type": "module",
-    "readme": "README.md",
-    "author": {
-        "name": "Needle",
-        "email": "help@needle.tools",
-        "url": "https://needle.tools/"
-    },
-    "repository": {
-        "type": "git",
-        "url": "git+https://github.com/needle-tools/needle-gltf-extensions.git"
-    },
-    "bin": {
-        "gltf-build-pipeline": "./dist/src/cli/index.js",
-        "needle-gltf": "./dist/src/cli/index.js"
-    },
-    "scripts": {
-        "start": "node scripts/index.mjs",
-        "dev": "npm-watch compile",
-        "compile": "tsc || exit 1",
-        "clear-caches": "node scripts/clear-caches.mjs",
-        "limit-caches": "node scripts/limit-caches.mjs",
-        "transform": "node scripts/index.mjs",
-        "transform:make-progressive": "node scripts/make-progressive.mjs",
-        "transform:pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
-        "pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
-        "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs"
-    },
-    "devDependencies": {
-        "@img/sharp-win32-x64": "^0.33.4",
-        "@types/node": "^18.7.23",
-        "npm-watch": "^0.13.0",
-        "typescript": "^4.9.5"
-    },
-    "optionalDependencies": {
-        "@img/sharp-linux-arm": "^0.33.4",
-        "@img/sharp-win32-x64": "^0.33.4",
-        "@img/sharp-darwin-arm64": "^0.33.4",
-        "@img/sharp-darwin-x64": "^0.33.4"
-    },
-    "dependencies": {
-        "@donmccurdy/caporal": "~0.0.10",
-        "@gltf-transform/cli": "^4.0.1",
-        "@gltf-transform/core": "^4.0.1",
-        "@gltf-transform/extensions": "^4.0.1",
-        "@gltf-transform/functions": "^4.0.1",
-        "command-exists": "^1.2.9",
-        "command-line-args": "^5.2.1",
-        "draco3dgltf": "^1.5.3",
-        "glob": "^7.2.0",
-        "meshoptimizer": "^0.18.1",
-        "micromatch": "^4.0.5",
-        "ndarray": "^1.0.19",
-        "ndarray-lanczos": "^0.3.0",
-        "ndarray-pixels": "^2.0.1",
-        "p-limit": "3.1.0",
-        "semver": "^7.3.7",
-        "sharp": "^0.33.4",
-        "tmp": "^0.2.1",
-        "uuid": "^9.0.1"
-    },
-    "watch": {
-        "compile": {
-            "patterns": [
-                "src"
-            ],
-            "extensions": "ts",
-            "quiet": false
-        }
-    }
-}

package/dist/src/cli/index.js

@@ -1,192 +0,0 @@
-#! /usr/bin/env node
-import { program } from '@donmccurdy/caporal';
-import { packGLTF } from "../scripts/index.js";
-import { printStats, writeFileStatsToFile } from '../utils/stats.js';
-import { getVersion } from '../utils/version.js';
-import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
-import { make_progressive } from '../transforms/needle_progressive.js';
-import { cacheSizeLimit, clearCache } from '../cache/cache.js';
-import { existsSync, statSync } from 'fs';
-import { foreachGLTF, getDirName } from '../utils/fileutils.js';
-import path, { dirname } from 'path';
-import { ERROR_CODES } from '../constants.js';
-// For testing / dev you can run `npm link` in the package directory
-// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
-program
-    .command("version", "Print current verison")
-    .action(async ({ logger }) => {
-    logger.info(getVersion());
-})
-    .command("clear-caches", "Clear all caches")
-    .help(`Clears all cached files
-Caches are limited to ${cacheSizeLimit} MB disc space by default.
-`)
-    .action(async ({ logger }) => {
-    logger.info("Clearing caches");
-    clearCache(logger);
-})
-    .command("compress", "Compress glTF, GLB or VRM files")
-    .help(`
-Compress a single glTF file. The input file is compressed and written to the output file.  
-`)
-    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
-    .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
-    .option("--usecase <usecase>", `The usecase for the compression allows you to determine how your asset is supposed to be used:\nThis will set compression and transform settings on a high level if no other settings are found.\nPossible options: [${UsecaseOptions.join(", ")}] e.g. \"--usecase ${UsecaseOptions[0]}\"`, { validator: program.STRING })
-    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
-    .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
-    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
-    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
-    .action(async ({ logger, args, options }) => {
-    logger.info("Transforming glTF file: " + args.input);
-    const stats = {
-        startTime: Date.now(),
-        totalGPUMemoryInMB: 0,
-        totalFilesProcessed: 0,
-        totalFileSizeInMB: 0,
-        totalFileSizeInMBBefore: 0,
-    };
-    const file = args.input.toString();
-    let config = getConfig(options.config?.toString(), file);
-    if (options.usecase) {
-        logger.info("Using usecase: " + options.usecase);
-        config = { ...createConfig(options.usecase.toString()), ...config, };
-        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
-            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
-        }
-        else {
-            config.usecase = options.usecase.toString();
-        }
-    }
-    const debug = options.debug === true;
-    const isDirectory = statSync(file).isDirectory();
-    const outDir = isDirectory ? file : dirname(file);
-    let outputIsDirectory = false;
-    if (statSync(file).isDirectory() && options.output) {
-        if (statSync(options.output.toString()).isDirectory()) {
-            outputIsDirectory = true;
-            if (!existsSync(options.output.toString())) {
-                logger.error("Output directory does not exist: " + options.output.toString());
-                return;
-            }
-        }
-        else {
-            logger.error("--output must be a directory if input is a directory");
-            return;
-        }
-    }
-    const file_stats = [];
-    const set = new Set();
-    await foreachGLTF(file, async (file) => {
-        let outputFile = args.output?.toString();
-        if (!outputFile?.length) {
-            outputFile = file;
-        }
-        if (outputIsDirectory) {
-            const filename = file.replaceAll("\\", "/").split("/").pop();
-            outputFile = args.output?.toString() + "/" + filename;
-        }
-        const res = await packGLTF(file, outputFile, {
-            config,
-            packedFiles: set,
-            stats,
-            debug,
-            useCache: options.cache !== false,
-            logger: logger,
-            file_stats: file_stats,
-        });
-        if (res === false) {
-            process.exit(ERROR_CODES.PACKING_FAILED);
-        }
-    });
-    printStats(stats, logger, config);
-    console.log("Stats: ", options);
-    if (options.stats) {
-        const statsFile = (isDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
-        writeFileStatsToFile(config, file_stats, statsFile);
-    }
-})
-    .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
-    .help(`
-This will produce multiple versions of the input file, each with a different level of optimization.   
-Each version will be compressed and written to the output directory.
-`)
-    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
-    // .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
-    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
-    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
-    .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
-    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
-    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
-    .action(async ({ logger, args, options }) => {
-    logger.info("Transforming glTF file: " + args.input);
-    const stats = {
-        startTime: Date.now(),
-        totalGPUMemoryInMB: 0,
-        totalFilesProcessed: 0,
-        totalFileSizeInMB: 0,
-        totalFileSizeInMBBefore: 0,
-    };
-    const file = args.input.toString();
-    let config = getConfig(options.config?.toString(), file);
-    if (options.usecase) {
-        logger.info("Using usecase: " + options.usecase);
-        config = { ...createConfig(options.usecase.toString()), ...config, };
-        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
-            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
-        }
-        else {
-            config.usecase = options.usecase.toString();
-        }
-    }
-    const isDirectory = statSync(file).isDirectory();
-    const outDir = isDirectory ? file : dirname(file);
-    const file_stats = [];
-    const set = new Set();
-    const useCache = options.cache !== false;
-    const debug = options.debug === true;
-    await foreachGLTF(file, async (file) => {
-        logger.info("→ Transform " + file);
-        const progressive_results = new Array();
-        await make_progressive({
-            path: file,
-            results: progressive_results,
-            config,
-            debug,
-            useCache,
-        }).catch(err => {
-            logger.error(err);
-            process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
-        });
-        const opts = {
-            config,
-            stats,
-            packedFiles: set,
-            debug,
-            useCache,
-            logger: logger,
-            file_stats,
-        };
-        for (const res of progressive_results) {
-            if (await packGLTF(res, res, opts) === false) {
-                process.exit(ERROR_CODES.PACKING_FAILED);
-            }
-        }
-        if (await packGLTF(file, file, opts) === false) {
-            process.exit(ERROR_CODES.PACKING_FAILED);
-        }
-    });
-    printStats(stats, logger, config);
-    if (options.stats) {
-        const statsFile = (isDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
-        writeFileStatsToFile(config, file_stats, statsFile);
-    }
-});
-program.run().catch(err => {
-    // Since we still invoke this package with paths when running npm scripts we need to ignore this error
-    // We might want to move this code here into an extra package.
-    if (err.message.includes("Unknown command") && !err.message.includes("Did you mean")) {
-        // ignore
-        return;
-    }
-    throw err;
-});