@needle-tools/gltf-build-pipeline 2.2.1-alpha → 2.2.1-alpha.1

package/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.2.1-alpha.1] - 2023-06-17
+- add: `--verbose` option to transform command
+- add: `stats` command to create stats.needle.json
+
 ## [2.2.1-alpha] - 2023-06-15
 - fix: issue with progressive mesh cache key calculation when running multiple transform processes with different configurations
 

package/dist/cli/index.js

@@ -1,13 +1,13 @@
 #! /usr/bin/env node
 import { program } from '@donmccurdy/caporal';
-import { packGLTF } from "../scripts/index.js";
+import { calculateStats, packGLTF } from "../scripts/index.js";
 import { printStats, writeFileStatsToFile } from '../utils/stats.js';
 import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache } from '../cache/cache.js';
 import { statSync } from 'fs';
-import { foreachGLTF, getDirName } from '../utils/fileutils.js';
+import { foreachGLTF, getDirName, isDirectory } from '../utils/fileutils.js';
 import path, { dirname } from 'path';
 import { ERROR_CODES } from '../constants.js';
 // For testing / dev you can run `npm link` in the package directory
@@ -24,6 +24,32 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
     .action(async ({ logger }) => {
     logger.info("Clearing caches");
     clearCache(logger);
+})
+    .command("stats", "Calculate glTF stats")
+    .help(`
+`)
+    .argument('<input>', "Path to a glTF, GLB or VRM file OR a directory")
+    .argument('<output>', "Output path for the stats", { default: null, validator: program.STRING })
+    .action(async ({ logger, args, options }) => {
+    logger.level = "info";
+    logger.info("Calculating stats for \"" + args.input + "\"");
+    const input = args.input.toString();
+    const filestats = new Array();
+    await foreachGLTF(input, async (file) => {
+        const stats = await calculateStats(file);
+        filestats.push(stats);
+    });
+    let output = args.ourput?.toString();
+    if (!output) {
+        if (isDirectory(input)) {
+            output = path.join(input, "stats.needle.json");
+        }
+        else {
+            output = input + ".needle.stats.json";
+        }
+    }
+    logger.info("Writing stats to: " + output);
+    writeFileStatsToFile(filestats, output);
 })
     .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
     .help(`
@@ -39,6 +65,7 @@ Each version will be compressed and written to the output directory.
     .option("--stats", "Output a stats file next to input file (or inside the input directory). It will be named as the input file/directory \"<input>.needle.stats.json\"", { validator: program.BOOLEAN, default: false })
     .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
     .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
+    .option("--verbose", "Enable for more verbose outout with \"--verbose True\"", { validator: program.BOOLEAN, default: false })
     .action(async ({ logger, args, options }) => {
     if (options.compress === false && options.progressive === false) {
         logger.error("You need to enable at least one of the options: --compress or --progressive");
@@ -69,7 +96,7 @@ Each version will be compressed and written to the output directory.
     const output = options.output?.toString() || file;
     const outputIsDirectory = statSync(output).isDirectory();
     const outDir = outputIsDirectory ? output : dirname(output);
-    const file_stats = [];
+    const file_stats = options.stats === true ? new Array : null;
     const set = new Set();
     const useCache = options.cache !== false;
     const debug = options.debug === true;
@@ -113,7 +140,7 @@ Each version will be compressed and written to the output directory.
     printStats(stats, logger, config);
     if (options.stats) {
         const statsFile = (outputIsDirectory ? (path.join(file, getDirName(file))) : file) + ".needle.stats.json";
-        writeFileStatsToFile(config, file_stats, statsFile);
+        writeFileStatsToFile(file_stats, statsFile);
     }
 });
 program.run().catch(err => {

package/dist/scripts/index.d.ts

@@ -1 +1,2 @@
 export * from "./pack-gltf.js";
+export * from "./stats.js";

package/dist/scripts/index.js

@@ -1 +1,2 @@
 export * from "./pack-gltf.js";
+export * from "./stats.js";

package/dist/scripts/pack-gltf.d.ts

@@ -1,14 +1,15 @@
 import { ILogger } from '@gltf-transform/core';
-import { FileStats, Stats } from "../utils/index.js";
+import { FileStats, ProcessStats } from "../utils/index.js";
 import { Config } from '../config/index.js';
 export type PackGLTFOptions = {
-    debug: boolean;
     useCache: boolean;
     packedFiles?: Set<string>;
-    stats?: Stats;
+    stats?: ProcessStats;
     file_stats: FileStats[] | null;
     config: Config;
     logger?: ILogger;
+    debug: boolean;
+    verbose?: boolean;
 };
 /** Compress a glTF file
  */

package/dist/scripts/pack-gltf.js

@@ -60,7 +60,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         });
         const document = await io.read(inputFile);
         /** @ts-ignore */
-        document.getLogger().verbosity = options.debug ? Verbosity.DEBUG : Verbosity.WARN;
+        document.getLogger().verbosity = options.debug || options.verbose ? Verbosity.DEBUG : Verbosity.WARN;
         let totalGPUTextureMemory = 0;
         /** @type {Array<import('@gltf-transform/core').Transform>} */
         const transforms = [
@@ -107,9 +107,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             stats.totalFileSizeInMBBefore += prevSizeInMB;
             stats.totalGPUMemoryInMB += gpuMemoryInMB;
         }
-        const filestats = createNewFilestats(outputFile, document, {
-            gpu_memory: totalGPUTextureMemory,
-        });
+        const filestats = createNewFilestats(outputFile, document);
         options.file_stats?.push(filestats);
         return true;
     }

package/dist/scripts/stats.d.ts

@@ -0,0 +1,2 @@
+import { FileStats } from "../utils/stats.js";
+export declare function calculateStats(inputfile: string): Promise<FileStats>;

package/dist/scripts/stats.js

@@ -0,0 +1,18 @@
+import { NodeIO } from "@gltf-transform/core";
+import { ALL_EXTENSIONS } from "@gltf-transform/extensions";
+import draco3d from 'draco3dgltf';
+import { MeshoptDecoder, MeshoptEncoder } from "meshoptimizer";
+import { createNewFilestats } from "../utils/stats.js";
+export async function calculateStats(inputfile) {
+    const io = new NodeIO()
+        .registerExtensions(ALL_EXTENSIONS)
+        .registerDependencies({
+        'draco3d.decoder': await draco3d.createDecoderModule(),
+        'draco3d.encoder': await draco3d.createEncoderModule(),
+        'meshopt.decoder': MeshoptDecoder,
+        'meshopt.encoder': MeshoptEncoder,
+    });
+    const document = await io.read(inputfile);
+    const stats = createNewFilestats(inputfile, document);
+    return stats;
+}

package/dist/transforms/needle_progressive.d.ts

@@ -5,6 +5,7 @@ declare interface ProgressiveOptions {
     results: Array<string>;
     config: Config;
     debug?: boolean;
+    verbose?: boolean;
     useCache?: boolean;
 }
 export declare function make_progressive(opts: ProgressiveOptions): Promise<boolean>;

package/dist/transforms/needle_progressive.js

@@ -65,7 +65,7 @@ async function onProcess(file, opts) {
         if (document) {
             const logger = document.getLogger();
             //@ts-ignore
-            logger.verbosity = opts.debug ? Verbosity.DEBUG : Verbosity.WARN;
+            logger.verbosity = opts.debug || opts.verbose ? Verbosity.DEBUG : Verbosity.WARN;
             let smallesTextureSize = 128;
             if (opts.config.usecase === "product") {
                 smallesTextureSize = 512;

package/dist/transforms/needle_toktx.js

@@ -45,11 +45,6 @@ export class NeedleToktxContext {
                 this.env.PATH = `${process.env.NEEDLE_TOKTX};${this.env.PATH}`;
             }
         }
-        else {
-            if (didLogEnvironmentInfo === false)
-                console.log("INFO: Environment variable 'NEEDLE_TOKTX' not set. Use 'NEEDLE_TOKTX' to specify the directory of the toktx installation (path/to/bin)");
-            didLogEnvironmentInfo = true;
-        }
         // fallback default installation directory:
         if (isUsingWindows) {
             const defaultPath = "C:\\Program Files\\KTX-Software\\bin";
@@ -59,10 +54,15 @@ export class NeedleToktxContext {
         // append default system paths
         if (isUsingWindows && process.env.Path?.length)
             this.env.PATH = `${this.env.PATH};${process.env.Path}`;
+        const ktxPaths = this.env.PATH.split(";").filter(p => p.toLocaleLowerCase().includes("ktx"));
         if (options.debug) {
-            const ktxPaths = this.env.PATH.split(";").filter(p => p.toLocaleLowerCase().includes("ktx"));
             console.log("INFO: Found these paths including ktx:\n\t" + ktxPaths.join("\n\t") + "");
         }
+        else if (ktxPaths?.length <= 0) {
+            if (didLogEnvironmentInfo === false)
+                console.log("INFO: Environment variable 'NEEDLE_TOKTX' not set. Use 'NEEDLE_TOKTX' to specify the directory of the toktx installation (path/to/bin)");
+            didLogEnvironmentInfo = true;
+        }
     }
     async prepare(document) {
         try {

package/dist/transforms/needle_webp.js

@@ -24,7 +24,7 @@ export class NeedleWebPContext {
             options.quality = 1;
         }
         // console.log(options);
-        console.log(`> Will compress texture ${texture.getName()} to webp with quality ${options.quality}`);
+        this.logger.info(`→ Will compress texture ${texture.getName()} to webp with quality ${options.quality}`);
         const buffer = await sharp(texture.getImage()).toFormat('webp', options).toBuffer();
         texture
             .setMimeType('image/webp')

package/dist/utils/fileutils.d.ts

@@ -3,3 +3,4 @@ export declare function getDirName(filepath: string): string;
 export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
 export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;
 export declare function replaceExtension(filepath: string, newExtension: string): string;
+export declare function isDirectory(path: string): boolean;

package/dist/utils/fileutils.js

@@ -132,3 +132,10 @@ export function replaceExtension(filepath, newExtension) {
     parts.pop();
     return parts.join(".") + "." + newExtension;
 }
+export function isDirectory(path) {
+    if (existsSync(path)) {
+        return statSync(path).isDirectory();
+    }
+    const hasExt = path.replaceAll("\\", "/").split("/").pop()?.includes(".");
+    return !hasExt;
+}

package/dist/utils/index.d.ts

@@ -6,3 +6,4 @@ export * from "./texture.js";
 export * from "./guid.js";
 export * from "./nodeio.js";
 export * from "./stats.js";
+export * from "./merge.js";

package/dist/utils/index.js

@@ -6,3 +6,4 @@ export * from "./texture.js";
 export * from "./guid.js";
 export * from "./nodeio.js";
 export * from "./stats.js";
+export * from "./merge.js";

package/dist/utils/merge.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Simple object check.
+ * @param item
+ * @returns {boolean}
+ */
+export declare function isObject(item: object): boolean;
+/**
+ * Deep merge two objects.
+ * @param target
+ * @param ...sources
+ */
+export declare function mergeDeep(target: any, ...sources: any[]): any;

package/dist/utils/merge.js

@@ -0,0 +1,31 @@
+/**
+ * Simple object check.
+ * @param item
+ * @returns {boolean}
+ */
+export function isObject(item) {
+    return (item && typeof item === 'object' && !Array.isArray(item));
+}
+/**
+ * Deep merge two objects.
+ * @param target
+ * @param ...sources
+ */
+export function mergeDeep(target, ...sources) {
+    if (!sources.length)
+        return target;
+    const source = sources.shift();
+    if (isObject(target) && isObject(source)) {
+        for (const key in source) {
+            if (isObject(source[key])) {
+                if (!target[key])
+                    Object.assign(target, { [key]: {} });
+                mergeDeep(target[key], source[key]);
+            }
+            else {
+                Object.assign(target, { [key]: source[key] });
+            }
+        }
+    }
+    return mergeDeep(target, ...sources);
+}

package/dist/utils/stats.d.ts

@@ -1,29 +1,53 @@
 import { Logger } from "@donmccurdy/caporal";
 import { Document, Logger as _Logger } from "@gltf-transform/core";
 import { Config } from "../config/index.js";
-export type Stats = {
+export type ProcessStats = {
     startTime: number;
     totalGPUMemoryInMB: number;
     totalFilesProcessed: number;
     totalFileSizeInMB: number;
     totalFileSizeInMBBefore: number;
 };
-export declare function printStats(stats: Stats, logger: Logger | _Logger, config: Config): void;
-export type FileStats = {
-    filename: string;
+export declare function printStats(stats: ProcessStats, logger: Logger | _Logger, config: Config): void;
+declare class Stats {
     /** Size in bytes */
     size: number;
-    /** Memory in bytes */
-    gpu_memory: number;
+    asset: {
+        version: string;
+        generator?: string;
+        copyright?: string;
+        extras: any;
+    };
     nodes: number;
     materials: number;
-    meshes: number;
-    textures: number;
-    vertex_count: number;
-    triangle_count: number;
-};
+    meshes: MeshInfo;
+    textures: TextureInfo;
+    constructor(document: Document, size: number);
+    static combine(others: Stats[]): Stats;
+}
+export declare class FileStats extends Stats {
+    readonly filename: string;
+    constructor(file: string, document: Document);
+}
+export declare class MeshInfo {
+    count: number;
+    primitives: number;
+    vertices: number;
+    triangles: number;
+    memory: number;
+    formats: string[];
+    addTo(other: MeshInfo): void;
+}
+export declare class TextureInfo {
+    count: number;
+    memory: number;
+    gpu_memory: number;
+    formats: string[];
+    addTo(other: TextureInfo): void;
+}
 /**
  * Create a new filestats object from a file and a document
  */
-export declare function createNewFilestats(file: string, document: Document, stats: Pick<FileStats, "gpu_memory">): FileStats;
-export declare function writeFileStatsToFile(config: Config, filestats: FileStats[], output_path: string): void;
+export declare function createNewFilestats(file: string, document: Document): FileStats;
+export declare function writeFileStatsToFile(filestats: FileStats[], output_path: string): string | null;
+export {};

package/dist/utils/stats.js

@@ -1,4 +1,5 @@
 import { getVersion } from "./version.js";
+import { Document, ImageUtils } from "@gltf-transform/core";
 import { getFileName } from "./fileutils.js";
 import { statSync, writeFileSync } from "fs";
 export function printStats(stats, logger, config) {
@@ -10,36 +11,156 @@ export function printStats(stats, logger, config) {
     else
         console.log(msg);
 }
+class Stats {
+    /** Size in bytes */
+    size;
+    asset;
+    nodes;
+    materials;
+    meshes;
+    textures;
+    constructor(document, size) {
+        const root = document.getRoot();
+        const asset = root.getAsset();
+        this.asset = {
+            generator: asset.generator,
+            version: asset.version,
+            copyright: asset.copyright,
+            extras: asset.extras,
+        };
+        this.size = size;
+        this.nodes = root.listNodes().length;
+        this.materials = root.listMaterials().length;
+        this.meshes = calculateMeshInfo(document);
+        this.textures = calculateTextureInfo(document);
+    }
+    static combine(others) {
+        const res = new Stats(new Document(), 0);
+        for (const other of others) {
+            res.size += other.size;
+            res.nodes += other.nodes;
+            res.materials += other.materials;
+            res.meshes.addTo(other.meshes);
+            res.textures.addTo(other.textures);
+        }
+        return res;
+    }
+}
+export class FileStats extends Stats {
+    filename;
+    constructor(file, document) {
+        super(document, statSync(file).size);
+        this.filename = getFileName(file);
+    }
+}
+export class MeshInfo {
+    count = 0;
+    primitives = 0;
+    vertices = 0;
+    triangles = 0;
+    memory = 0;
+    formats = [];
+    addTo(other) {
+        this.count += other.count;
+        this.primitives += other.primitives;
+        this.vertices += other.vertices;
+        this.triangles += other.triangles;
+        this.memory += other.memory;
+        this.formats = Array.from(new Set(this.formats.concat(other.formats)));
+    }
+}
+export class TextureInfo {
+    count = 0;
+    memory = 0;
+    gpu_memory = 0;
+    formats = [];
+    addTo(other) {
+        this.count += other.count;
+        this.memory += other.memory;
+        this.gpu_memory += other.gpu_memory;
+        this.formats = Array.from(new Set(this.formats.concat(other.formats)));
+    }
+}
 /**
  * Create a new filestats object from a file and a document
  */
-export function createNewFilestats(file, document, stats) {
-    const filename = getFileName(file);
-    const root = document.getRoot();
-    const filestats = {
-        filename,
-        size: statSync(file).size,
-        gpu_memory: Math.round(stats.gpu_memory),
-        nodes: root.listNodes().length,
-        materials: root.listMaterials().length,
-        meshes: root.listMeshes().length,
-        textures: root.listTextures().length,
-        vertex_count: 0,
-        triangle_count: 0,
-    };
-    // Count vertices
-    for (const mesh of root.listMeshes()) {
-        filestats.vertex_count += mesh.listPrimitives().reduce((acc, p) => acc + p.getAttribute('POSITION').getCount(), 0);
-    }
-    filestats.triangle_count = filestats.vertex_count / 3;
+export function createNewFilestats(file, document) {
+    const filestats = new FileStats(file, document);
     return filestats;
 }
-export function writeFileStatsToFile(config, filestats, output_path) {
+export function writeFileStatsToFile(filestats, output_path) {
     if (output_path == undefined) {
         console.error("No stats output path provided");
-        return;
+        return null;
     }
-    console.log("Writing file stats to: " + output_path);
-    const json = JSON.stringify(filestats);
+    const res = {
+        sum: Stats.combine(filestats),
+        files: filestats,
+    };
+    const json = JSON.stringify(res);
     writeFileSync(output_path, json);
+    return json;
+}
+function calculateMeshInfo(document) {
+    const root = document.getRoot();
+    const meshes = root.listMeshes();
+    const res = new MeshInfo();
+    res.count = meshes.length;
+    for (const mesh of root.listMeshes()) {
+        const primitives = mesh.listPrimitives();
+        res.primitives += primitives.length;
+        for (const prim of primitives) {
+            const positions = prim.getAttribute('POSITION');
+            const indices = prim.getIndices();
+            if (positions) {
+                res.vertices += positions.getCount();
+            }
+            if (indices) {
+                res.triangles += indices.getCount() / 3;
+            }
+            else {
+                res.triangles += positions.getCount() / 3;
+            }
+            const semantics = prim.listSemantics();
+            for (const semantic of semantics) {
+                const accessor = prim.getAttribute(semantic);
+                if (accessor) {
+                    const buffer = accessor.getByteLength();
+                    res.memory += buffer;
+                }
+            }
+        }
+    }
+    const extensions = root.listExtensionsUsed();
+    for (const ext of extensions) {
+        if (ext.extensionName.startsWith("EXT_meshopt_compression")) {
+            res.formats.push("meshopt");
+        }
+        else if (ext.extensionName.startsWith("KHR_draco_mesh_compression")) {
+            res.formats.push("draco");
+        }
+    }
+    // filter formats so we only have unique values
+    res.formats = Array.from(new Set(res.formats));
+    return res;
+}
+function calculateTextureInfo(document) {
+    const root = document.getRoot();
+    const textures = root.listTextures();
+    const res = new TextureInfo();
+    res.count = textures.length;
+    res.formats = textures.map(t => t.getMimeType());
+    res.gpu_memory = 0;
+    res.memory = 0;
+    for (const texture of textures) {
+        const image = texture.getImage();
+        const mimetype = texture.getMimeType();
+        if (image && mimetype) {
+            res.gpu_memory += ImageUtils.getVRAMByteLength(image, mimetype);
+            res.memory += image.byteLength;
+        }
+    }
+    // filter formats so we only have unique values
+    res.formats = Array.from(new Set(res.formats));
+    return res;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.2.1-alpha",
+  "version": "2.2.1-alpha.1",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",