@needle-tools/gltf-build-pipeline 1.6.0-alpha → 1.6.0-alpha.2

package/CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [1.6.0-alpha] - 2023-05-27
+## [1.6.0-alpha.2] - 2023-05-27
 - add: `gltf-build-pipeline` commandline support with `transform` and `compress` (pack-gltf) options.
 
 ## [1.5.0-alpha.12] - 2023-05-20

package/dist/cache/cache.d.ts

@@ -5,6 +5,7 @@ import { Logger } from "@donmccurdy/caporal";
  * This will remove all files in the cache directory.
  */
 export declare function clearCache(logger?: Logger): void;
+export declare const cacheSizeLimit = 1024;
 /**
  * Ensure that the cache size does not exceed the given maximum size in MB.
  * @param maxSizeMB The maximum size in MB.

package/dist/cache/cache.js

@@ -31,11 +31,12 @@ export function clearCache(logger) {
     else
         logger.info(`Cleared cache at ${dir}`);
 }
+export const cacheSizeLimit = 1024;
 /**
  * Ensure that the cache size does not exceed the given maximum size in MB.
  * @param maxSizeMB The maximum size in MB.
  */
-export function limitCacheSize(maxSizeMB = 1024) {
+export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
     const dir = getCacheDirectory(true);
     console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}...`);
     let totalSize = 0;

package/dist/cli/index.js

@@ -5,7 +5,8 @@ import { printStats } from '../utils/stats.js';
 import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { make_progressive } from '../transforms/needle_progressive.js';
-import { clearCache } from '../cache/cache.js';
+import { cacheSizeLimit, clearCache } from '../cache/cache.js';
+import { foreachGLTF } from '../utils/fileutils.js';
 // For testing / dev you can run `npm link` in the package directory
 // For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
 program
@@ -14,8 +15,8 @@ program
     logger.info(getVersion());
 })
     .command("clear-caches", "Clear all caches")
-    .help(`
-Clear all caches.
+    .help(`Clears all cached files
+Caches are limited to ${cacheSizeLimit} MB disc space by default.
 `)
     .action(async ({ logger }) => {
     logger.info("Clearing caches");
@@ -25,8 +26,8 @@ Clear all caches.
     .help(`
 Compress a single glTF file. The input file is compressed and written to the output file.  
 `)
-    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed")
-    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Options are ${UsecaseOptions.join(", ")}`, { validator: program.STRING })
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}] e.g. \`--usecase ${UsecaseOptions[0]}\"`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
     .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
     .action(async ({ logger, args, options }) => {
@@ -41,28 +42,36 @@ Compress a single glTF file. The input file is compressed and written to the out
     let config = getConfig(options.config?.toString());
     if (options.usecase) {
         logger.info("Using usecase: " + options.usecase);
-        config = { ...config, ...createConfig(options.usecase.toString()) };
+        config = { ...createConfig(options.usecase.toString()), ...config, };
+        if (config.usecase != "unknown" && config.usecase != options.usecase.toString()) {
+            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
+        }
+        else {
+            config.usecase = options.usecase.toString();
+        }
     }
     const file = args.input.toString();
     const set = new Set();
-    await packGLTF(file, options.output?.toString() || file, {
-        config,
-        packedFiles: set,
-        stats,
-        debug: false,
-        useCache: true,
+    await foreachGLTF(file, async (file) => {
+        await packGLTF(file, options.output?.toString() || file, {
+            config,
+            packedFiles: set,
+            stats,
+            debug: false,
+            useCache: true,
+            logger: logger,
+        });
     });
     printStats(stats);
 })
-    .command("transform", "Transform a glTF file")
+    .command("transform", "Transform and optimize glTF, GLB or VRM files")
     .help(`
 This will produce multiple versions of the input file, each with a different level of optimization.   
 Each version will be compressed and written to the output directory.
 `)
-    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed")
-    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Options are ${UsecaseOptions.join(", ")}`, { validator: program.STRING })
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
-    // .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
     .action(async ({ logger, args, options }) => {
     logger.info("Transforming glTF file: " + args.input);
     const stats = {
@@ -75,28 +84,41 @@ Each version will be compressed and written to the output directory.
     let config = getConfig(options.config?.toString());
     if (options.usecase) {
         logger.info("Using usecase: " + options.usecase);
-        config = { ...config, ...createConfig(options.usecase.toString()) };
+        config = { ...createConfig(options.usecase.toString()), ...config, };
+        if (config.usecase != "unknown" && config.usecase != options.usecase.toString()) {
+            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
+        }
+        else {
+            config.usecase = options.usecase.toString();
+        }
     }
     const file = args.input.toString();
-    const progressive_results = new Array();
-    await make_progressive({
-        path: file,
-        results: progressive_results,
-        config,
-        debug: false,
-        useCache: true,
+    const set = new Set();
+    await foreachGLTF(file, async (file) => {
+        logger.info("> Transform " + file);
+        const progressive_results = new Array();
+        await make_progressive({
+            path: file,
+            results: progressive_results,
+            config,
+            debug: false,
+            useCache: true,
+        });
+        const opts = {
+            config,
+            stats,
+            packedFiles: set,
+            debug: false,
+            useCache: true,
+            logger: logger,
+        };
+        for (const res of progressive_results) {
+            logger.info("> Compress " + res);
+            await packGLTF(res, res, opts);
+        }
+        logger.info("> Compress " + file);
+        await packGLTF(file, file, opts);
     });
-    console.log("Progressive results: ", progressive_results);
-    const opts = {
-        config,
-        stats,
-        debug: false,
-        useCache: true,
-    };
-    for (const prog of progressive_results) {
-        await packGLTF(prog, prog, opts);
-    }
-    await packGLTF(file, file, opts);
     printStats(stats);
 });
 program.run();

package/dist/config/index.d.ts

@@ -1,4 +1,4 @@
-declare type Usecase = "unknown" | "product" | "game" | "metaverse";
+declare type Usecase = "unknown" | "product" | "world";
 export declare const UsecaseOptions: string[];
 /**
  * The config object contains information about how a file should be processed.

package/dist/config/index.js

@@ -1,5 +1,5 @@
 import { existsSync, readFileSync } from "fs";
-export const UsecaseOptions = ["product", "game", "metaverse"];
+export const UsecaseOptions = ["product"];
 const defaultConfig = {
     usecase: "unknown",
 };

package/dist/scripts/pack-gltf.d.ts

@@ -1,3 +1,4 @@
+import { ILogger } from '@gltf-transform/core';
 import { Stats } from "../utils/index.js";
 import { Config } from '../config/index.js';
 export type PackGLTFOptions = {
@@ -6,6 +7,7 @@ export type PackGLTFOptions = {
     packedFiles?: Set<string>;
     stats?: Stats;
     config?: Config;
+    logger?: ILogger;
 };
 /** Compress a glTF file
  */

package/dist/scripts/pack-gltf.js

@@ -1,5 +1,5 @@
 import path from 'path';
-import { NodeIO } from '@gltf-transform/core';
+import { Logger, NodeIO, Verbosity } from '@gltf-transform/core';
 import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
 import draco3d from 'draco3dgltf';
 import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
@@ -7,18 +7,20 @@ import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 import { resample } from '@gltf-transform/functions';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, } from '../extensions/index.js';
 import { isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
-import { writeNodeIO, getVersion } from "../utils/index.js";
+import { writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
 import { getConfig } from '../config/index.js';
-console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`);
 /** Compress a glTF file
  */
 export async function packGLTF(inputFile, outputFile, options) {
+    if (!options.logger) {
+        options.logger = new Logger(Verbosity.INFO);
+    }
+    const { debug, stats, packedFiles, useCache, logger } = options;
     if (!existsSync(inputFile)) {
-        console.error("WARN: can not pack, file does not exist: " + inputFile);
+        logger.error("WARN: can not pack, file does not exist: " + inputFile);
         return;
     }
-    const { debug, stats, packedFiles, useCache } = options;
     if (debug && inputFile.includes(".debug.")) {
         return;
     }
@@ -32,7 +34,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         cacheKey = "packed-" + hash;
         const cached = tryGetFromCache(cacheKey);
         if (cached) {
-            console.log(`→ Loaded compressed file from cache: \"${path.basename(inputFile)}\"`);
+            logger.info(`→ Loaded compressed file from cache: \"${path.basename(inputFile)}\"`);
             writeFileSync(inputFile, cached);
             return;
         }
@@ -45,7 +47,7 @@ export async function packGLTF(inputFile, outputFile, options) {
     try {
         const config = options.config || getConfig();
         const prevSizeInMB = statSync(inputFile).size / 1024 / 1024;
-        console.log(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
+        logger.info(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
         // const isProgressiveAsset = isLOD(file);
         const startTime = Date.now();
         const io = new NodeIO()
@@ -63,7 +65,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         });
         const document = await io.read(inputFile);
         /** @ts-ignore */
-        document.getLogger().verbosity = 0;
+        document.getLogger().verbosity = Verbosity.ERROR;
         let totalGPUTextureMemmory = 0;
         /** @type {Array<import('@gltf-transform/core').Transform>} */
         const transforms = [
@@ -101,7 +103,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         const sizeInMB = statSync(outputFile).size / 1024 / 1024;
         const filename = path.basename(outputFile);
         const gpuMemoryInMB = totalGPUTextureMemmory / 1024 / 1024;
-        console.log(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${outputFile}`);
+        logger.info(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${outputFile}`);
         if (stats) {
             stats.totalFilesProcessed++;
             stats.totalFileSizeInMB += sizeInMB;
@@ -111,7 +113,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         return true;
     }
     catch (err) {
-        console.error("ERR: packing " + inputFile + " failed: " + err.message);
+        logger.error("ERR: packing " + inputFile + " failed: " + err.message);
         if (debug)
             throw err;
         return false;

package/dist/transforms/needle_progressive.d.ts

@@ -9,7 +9,7 @@ declare interface ProgressiveOptions {
 }
 export declare function make_progressive(opts: ProgressiveOptions): Promise<void>;
 export declare function createTransform(name: string, fn: Transform): Transform;
-export declare function make_progressive_textures(filePath: string, _options?: TextureResizeOptions, opts?: ProgressiveOptions): Transform;
+export declare function make_progressive_textures(filePath: string, _options: TextureResizeOptions, opts: ProgressiveOptions): Transform;
 /** Options for the {@link textureResize} function. */
 export interface TextureResizeOptions {
     /**

package/dist/transforms/needle_progressive.js

@@ -69,11 +69,15 @@ async function onProcess(file, opts) {
                     return;
                 }
             }
+            let smallesTextureSize = 128;
+            if (opts.config.usecase === "product") {
+                smallesTextureSize = 512;
+            }
             //@ts-ignore
             document.getLogger().verbosity = 0;
             const transforms = [
                 needle_asset(),
-                make_progressive_textures(file, { size: [256, 256] }, opts),
+                make_progressive_textures(file, { size: [smallesTextureSize, smallesTextureSize] }, opts),
                 sparse(),
             ];
             // TODO: progressive meshes aren't supported yet for VRMs
@@ -143,14 +147,14 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             let settings = texture.getExtension(NEEDLE_progressive_texture_settings.EXTENSION_NAME);
             if (!settings) {
                 const size = texture.getSize();
-                if (size?.length >= 2 && size[0] <= 128 && size[1] <= 128) {
-                    logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found and the texture is already < 128x128`);
+                if (size?.length >= 2 && size[0] <= options.size[0] && size[1] <= options.size[1]) {
+                    logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found and the texture is already <= ${size[0]}x${size[1]}`);
                     continue;
                 }
                 else {
                     settings = {
                         guid: generateGuid(filename + "_texture_" + index.toString()),
-                        maxSize: 128,
+                        maxSize: _options.size[0],
                     };
                     logger.debug(`[${NAME}] No settings found for texture[${index}] - will use default settings (maxSize: ${settings.maxSize}, guid: ${settings.guid})`);
                 }
@@ -360,7 +364,10 @@ export function make_progressive_meshes(filePath, _options, opts) {
         console.log(`[${NAME}]: Begin generating progressive meshes for \"${filename}\"`);
         let index = 0;
         // the lowest LOD (last entry) will be saved in the original file
-        const MAX_LODS = 6;
+        let MAX_LODS = 6;
+        if (opts.config.usecase === "product") {
+            MAX_LODS = 3;
+        }
         const LODS = [
             {
                 ratio: 1,
@@ -369,10 +376,13 @@ export function make_progressive_meshes(filePath, _options, opts) {
             },
         ];
         for (let l = 0; l < MAX_LODS; l++) {
-            const t = l / MAX_LODS;
+            // const t = l / MAX_LODS;
             const latest = LODS[LODS.length - 1];
             const ratio = latest.ratio * .5;
-            const error = Math.min(.013 * (l + 1), 0.05);
+            let error = Math.min(.013 * (l + 1), 0.05);
+            if (opts.config.usecase === "product") {
+                error *= .5;
+            }
             LODS.push({
                 ratio: ratio,
                 error,
@@ -626,6 +636,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
         mesh["needle:mesh-hash"] = hash;
     }
     hash += getHash(lodLevel);
+    hash += getHash(opts.config);
     if (opts.useCache !== false) {
         cacheKey = `mesh_lod_${level}_${hash}`;
         const cachedValue = tryGetFromCache(cacheKey);

package/dist/utils/stats.d.ts

@@ -1,3 +1,4 @@
+import { Logger } from "@donmccurdy/caporal";
 export type Stats = {
     startTime: number;
     totalGPUMemoryInMB: number;
@@ -5,4 +6,4 @@ export type Stats = {
     totalFileSizeInMB: number;
     totalFileSizeInMBBefore: number;
 };
-export declare function printStats(stats: any): void;
+export declare function printStats(stats: Stats, logger?: Logger): void;

package/dist/utils/stats.js

@@ -1,6 +1,10 @@
 import { getVersion } from "./version.js";
-export function printStats(stats) {
+export function printStats(stats, logger) {
     const duration = Date.now() - stats.startTime;
     const durationInMinutes = duration / 1000 / 60;
-    console.log(`[Needle Build Pipeline] v${getVersion()} — Finished packing ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`);
+    const msg = `[Needle Build Pipeline] v${getVersion()} — Finished packing ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
+    if (logger)
+        logger.info(msg);
+    else
+        console.log(msg);
 }

package/dist/utils/version.js

@@ -1,18 +1,32 @@
-import { existsSync, readFileSync } from 'fs';
+import { existsSync, readFileSync, statSync } from 'fs';
 let version = undefined;
 let isLocal = false;
 export function getVersion(marklocal = true) {
     if (version === undefined) {
-        version = null;
-        const path = process.cwd() + "/package.json";
-        isLocal = !path.includes("node_modules");
-        if (existsSync(path)) {
-            const packageJson = readFileSync(path, "utf8");
-            const packageVersion = JSON.parse(packageJson).version;
-            version = packageVersion ?? null;
+        const scriptLocation = process.argv[1].replaceAll("\\", "/");
+        const path = scriptLocation.split('/');
+        let packageJsonPath = null;
+        while (path.length) {
+            const dirname = path.join('/');
+            if (statSync(dirname || '/').isDirectory()) {
+                const filename = dirname + '/package.json';
+                if (existsSync(filename)) {
+                    packageJsonPath = filename;
+                    break;
+                }
+            }
+            path.pop();
+        }
+        if (packageJsonPath) {
+            isLocal = !path.includes("node_modules");
+            if (existsSync(packageJsonPath)) {
+                const packageJson = readFileSync(packageJsonPath, "utf8");
+                const packageVersion = JSON.parse(packageJson).version;
+                version = packageVersion ?? null;
+            }
+            else
+                console.log("Could not find package.json at " + packageJsonPath);
         }
-        else
-            console.log("Could not find package.json at " + path);
     }
     let res = version;
     if (marklocal && isLocal)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.6.0-alpha",
+  "version": "1.6.0-alpha.2",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",