@needle-tools/gltf-build-pipeline 2.9.5-release.0dfa267 → 2.10.0-next

package/dist/cache/cache.d.ts

@@ -0,0 +1,55 @@
+/// <reference types="node" resolution-mode="require"/>
+import { ExtensibleProperty } from "@gltf-transform/core";
+import { Logger } from "@donmccurdy/caporal";
+/**
+ * Clear the cache.
+ * This will remove all files in the cache directory.
+ */
+export declare function clearCache(logger?: Logger): void;
+export declare const cacheSizeLimit = 1024;
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
+export declare function limitCacheSize(maxSizeMB?: number): void;
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
+export declare function addToCache(key: string, bytes: Uint8Array | Buffer): void;
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
+export declare function tryGetFromCache(key: string): Uint8Array | null;
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export declare function addMetaToCache(key: string, meta: object): void;
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export declare function tryGetMetaFromCache(key: string, ignoreVersion?: boolean): object | null;
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
+export declare function getKey(property: ExtensibleProperty): string;
+export declare function getKeyWithHash(property: ExtensibleProperty): {
+    key: string;
+    hash: string;
+};
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @param level The current level of recursion.
+ * @returns The hash as a number.
+ */
+export declare function getHash(property: ExtensibleProperty | object, level?: number): string;

package/dist/cache/cache.js

@@ -0,0 +1,289 @@
+import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
+import tmp from 'tmp';
+import fs, { statSync } from 'fs';
+import path from 'path';
+import { getVersion } from "../utils/version.js";
+// import * as CHECKDISCSPACE from 'check-disk-space'
+/**
+ * Get the available space on the cache directory in bytes.
+ */
+// export async function getCacheAvailableSpaceOnDisc(): Promise<number> {
+//     try {
+//         const dir = getCacheDirectory();
+//         const res = await CHECKDISCSPACE.default(dir).catch(() => { return { free: 0 } });
+//         return res.free;
+//     }
+//     catch (e) {
+//         console.error(`Error checking disk space: ${e}`);
+//     }
+//     return 0;
+// }
+const buildPipelineVersion = getVersion(false);
+const debugLogs = process.argv.includes("--debug");
+/**
+ * Clear the cache.
+ * This will remove all files in the cache directory.
+ */
+export function clearCache(logger) {
+    const dir = getCacheDirectory(true);
+    fs.rmSync(dir, { recursive: true });
+    if (!logger)
+        console.log(`Cleared cache at ${dir}`);
+    else
+        logger.info(`Cleared cache at ${dir}`);
+}
+export const cacheSizeLimit = 1024;
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
+export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
+    const dir = getCacheDirectory(true);
+    console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}...`);
+    let totalSize = 0;
+    const files = new Array();
+    function recurseDirectories(dir) {
+        fs.readdirSync(dir).forEach(file => {
+            const path = `${dir}/${file}`;
+            const stats = statSync(path);
+            if (stats.isDirectory()) {
+                recurseDirectories(path);
+            }
+            else {
+                totalSize += stats.size;
+                files.push({ path, size: stats.size });
+            }
+        });
+    }
+    recurseDirectories(dir);
+    const maxSizeInBytes = maxSizeMB * 1024 * 1024;
+    console.log(`Current cache size is ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
+    if (totalSize < maxSizeInBytes)
+        return;
+    files.sort((a, b) => {
+        const mtimeA = fs.statSync(a.path).mtimeMs;
+        const mtimeB = fs.statSync(b.path).mtimeMs;
+        return mtimeA - mtimeB;
+    });
+    const targetSize = maxSizeInBytes * 0.9;
+    // remove oldest files first
+    let filesRemoved = 0;
+    while (totalSize > targetSize) {
+        const file = files.shift();
+        if (!file)
+            break;
+        if (fs.statSync(file.path).isDirectory()) {
+            continue;
+        }
+        filesRemoved++;
+        fs.rmSync(file.path);
+        totalSize -= file.size;
+    }
+    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB (${filesRemoved} files removed)`);
+}
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
+export function addToCache(key, bytes) {
+    const dir = getCacheDirectory();
+    const path = `${dir}/${key}.bin`;
+    if (debugLogs)
+        console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
+    if (bytes instanceof Buffer)
+        bytes = new Uint8Array(bytes);
+    fs.writeFileSync(path, bytes);
+}
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
+export function tryGetFromCache(key) {
+    const dir = getCacheDirectory();
+    const path = `${dir}/${key}.bin`;
+    if (fs.existsSync(path)) {
+        const bytes = fs.readFileSync(path);
+        if (bytes instanceof Buffer)
+            return new Uint8Array(bytes);
+        return bytes;
+    }
+    return null;
+}
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export function addMetaToCache(key, meta) {
+    const dir = getCacheDirectory();
+    const metaPath = `${dir}/${key}.meta`;
+    meta['needle-build-pipeline:version'] = buildPipelineVersion;
+    fs.writeFileSync(metaPath, JSON.stringify(meta));
+}
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export function tryGetMetaFromCache(key, ignoreVersion = false) {
+    try {
+        const dir = getCacheDirectory();
+        const path = `${dir}/${key}.meta`;
+        if (fs.existsSync(path)) {
+            const bytes = fs.readFileSync(path);
+            if (bytes?.length > 0) {
+                const text = bytes.toString('utf-8');
+                if (text) {
+                    const obj = JSON.parse(text);
+                    if (ignoreVersion)
+                        return obj;
+                    if (obj['needle-build-pipeline:version'] === buildPipelineVersion)
+                        return obj;
+                }
+            }
+        }
+        return null;
+    }
+    catch (e) {
+        console.error(`Error reading cache meta: ${e.message}`);
+        return null;
+    }
+}
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
+export function getKey(property) {
+    let prefix = "prop";
+    if (property instanceof Texture) {
+        prefix = "image";
+    }
+    let hash = getHash(property);
+    const key = `${prefix}-${hash}`;
+    return key;
+}
+export function getKeyWithHash(property) {
+    let prefix = "prop";
+    if (property instanceof Texture) {
+        prefix = "image";
+    }
+    let hash = getHash(property);
+    const key = `${prefix}-${hash}`;
+    return { key, hash };
+}
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @param level The current level of recursion.
+ * @returns The hash as a number.
+ */
+export function getHash(property, level = 0) {
+    let hashStr = "";
+    let hash = 0;
+    if (property instanceof ExtensibleProperty) {
+        const extensions = property.listExtensions();
+        const extras = property.getExtras();
+        // const slots = listTextureSlots(doc, texture);
+        // const channels = getTextureChannelMask(doc, texture);
+        hash += hashObject({
+            extensions,
+            extras,
+        });
+        if (property instanceof Texture) {
+            const image = property.getImage();
+            const bytes = image?.buffer;
+            if (bytes)
+                hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
+        }
+        else if (property instanceof Mesh) {
+            const weights = property.getWeights();
+            hash += hashObject(weights);
+            const primitives = property.listPrimitives();
+            for (const prim of primitives) {
+                hash += hashString(getHash(prim, level + 1));
+            }
+        }
+        else if (property instanceof Primitive) {
+            const indices = property.getIndices();
+            const attributes = property.listAttributes();
+            const targets = property.listTargets();
+            // TODO: hash buffer view if possible
+            const semantics = property.listSemantics();
+            const semanticsNames = semantics.map(s => {
+                return {
+                    name: s,
+                };
+            });
+            hash += hashObject({
+                indices,
+                attributes,
+                targets,
+                semantics: semanticsNames
+            });
+            if (indices) {
+                const indexBuffer = indices.getArray();
+                if (indexBuffer) {
+                    hash += hashBuffer(indexBuffer.buffer, indexBuffer.byteOffset, indexBuffer.byteLength);
+                }
+            }
+            // We can not pass the whole buffer into hashObject since very large meshes will cause errors when trying to stringify
+            // See https://linear.app/needle/issue/NE-6414
+            for (const sem of semantics) {
+                const arr = property.getAttribute(sem)?.getArray();
+                if (arr) {
+                    hash += hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength);
+                }
+            }
+        }
+    }
+    else {
+        hash += hashObject(property);
+    }
+    return hashStr + hash;
+}
+const cleanBuildPipelineVersion = getVersion(false).replace(/[^a-zA-Z0-9]/g, "_");
+function getCacheDirectory(root) {
+    const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
+    if (!fs.existsSync(tmpDir)) {
+        fs.mkdirSync(tmpDir, { recursive: true });
+    }
+    return tmpDir;
+}
+function hashObject(obj) {
+    if (obj instanceof Accessor) {
+        const buffer = obj.getArray();
+        return hashBuffer(buffer.buffer);
+    }
+    else if (obj == null) {
+        return 0;
+    }
+    try {
+        const str = JSON.stringify(obj);
+        return hashString(str);
+    }
+    catch (err) {
+        console.error(err);
+        debugger;
+        throw err;
+    }
+}
+function hashBuffer(buffer, offset, length) {
+    let hash = 0;
+    const view = new Uint8Array(buffer, offset, length);
+    for (let i = 0; i < view.length; i++) {
+        hash = ((hash << 5) - hash) + view[i];
+        hash |= 0;
+    }
+    return hash;
+}
+function hashString(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = ((hash << 5) - hash) + str.charCodeAt(i);
+        hash |= 0;
+    }
+    return hash;
+}

package/dist/cache/index.d.ts

@@ -0,0 +1 @@
+export * from "./cache.js";

package/dist/cache/index.js

@@ -0,0 +1 @@
+export * from "./cache.js";

package/dist/cli/index.d.ts

@@ -0,0 +1,2 @@
+#! /usr/bin/env node
+export {};

package/dist/cli/index.js

@@ -0,0 +1,189 @@
+#! /usr/bin/env node
+import { program } from '@donmccurdy/caporal';
+import { calculateStats, packGLTF } from "../scripts/index.js";
+import { printStats, writeFileStatsToFile } from '../utils/stats.js';
+import { getVersion } from '../utils/version.js';
+import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
+import { isLOD, make_progressive } from '../transforms/needle_progressive.js';
+import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
+import { existsSync } from 'fs';
+import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
+import path, { resolve } from 'path';
+import { ERROR_CODES } from '../constants.js';
+// For testing / dev you can run `npm link` in the package directory
+// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
+program
+    .command("version", "Print current verison")
+    .action(async ({ logger }) => {
+    logger.info(getVersion());
+})
+    .command("clear-caches", "Clear all caches")
+    .help(`Clears all cached files
+Caches are limited to ${cacheSizeLimit} MB disc space by default.
+`)
+    .action(async ({ logger }) => {
+    logger.info("Clearing caches");
+    clearCache(logger);
+})
+    .command("stats", "Calculate glTF stats")
+    .help(`
+`)
+    .argument('<input>', "Path to a glTF, GLB or VRM file OR a directory")
+    .argument('<output>', "Output path for the stats", { default: null, validator: program.STRING })
+    .action(async ({ logger, args, options }) => {
+    logger.level = "info";
+    logger.info("Calculating stats for \"" + args.input + "\"");
+    const input = args.input.toString();
+    const filestats = new Array();
+    await foreachGLTF(input, async (file) => {
+        const stats = await calculateStats(file);
+        filestats.push(stats);
+    });
+    let output = args.ourput?.toString();
+    if (!output) {
+        if (isDirectory(input)) {
+            output = path.join(input, "stats.needle.json");
+        }
+        else {
+            output = input + ".needle.stats.json";
+        }
+    }
+    logger.info("Writing stats to: " + output);
+    writeFileStatsToFile(filestats, output);
+})
+    .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
+    .help(`
+This will produce multiple versions of the input file, each with a different level of optimization.   
+Each version will be compressed and written to the output directory.
+`)
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
+    .argument('<output>', "The output file or directory. It can be an absolute path or a relative path to the input file. If not provided, the input file will be overwritten.", { default: null })
+    .option("--compress", `When enabled files will be compressed`, { validator: program.BOOLEAN, default: true })
+    .option("--progressive", `When enabled files will processed to be progressively loaded`, { validator: program.BOOLEAN, default: true })
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
+    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
+    .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
+    .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
+    .option("--verbose", "Enable for more verbose outout with \"--verbose True\"", { validator: program.BOOLEAN, default: false })
+    .action(async ({ logger, args, options }) => {
+    if (options.compress === false && options.progressive === false) {
+        logger.error("You need to enable at least one of the options: --compress or --progressive");
+        return;
+    }
+    logger.info("Transforming glTF file: " + args.input);
+    const stats = {
+        startTime: Date.now(),
+        totalGPUMemoryInMB: 0,
+        totalFilesProcessed: 0,
+        totalFileSizeInMB: 0,
+        totalFileSizeInMBBefore: 0,
+    };
+    const input = args.input.toString();
+    let config = getConfig(options.config?.toString(), input, {
+        logger
+    });
+    if (options.usecase) {
+        logger.info("Using usecase: " + options.usecase);
+        config = { ...createConfig(options.usecase.toString()), ...config, };
+        if (config.usecase != "default" && config.usecase != options.usecase.toString()) {
+            logger.warn(`Conflicting usecase. Your specific config at \"${options.config}\" has a different usecase than the one provided by the CLI: \"${options.usecase}\". Using the usecase from the config file.`);
+        }
+        else {
+            config.usecase = options.usecase.toString();
+        }
+    }
+    const inputIsDirectory = isDirectory(input);
+    let output = args.output?.toString();
+    // If the input path is a filepath and no specific output is defined we set output to the input file
+    if (!inputIsDirectory && !output) {
+        output = input;
+    }
+    const outputIsDirectory = output && isDirectory(output);
+    if (inputIsDirectory && output && !outputIsDirectory) {
+        logger.error("When <input> is a directory then <output> must also be a directory");
+        process.exit(ERROR_CODES.INVALID_ARGS);
+    }
+    // Handle the output bath being relative to the input path
+    if (output && path.isAbsolute(output) === false) {
+        const base = ensureIsDirectory(input);
+        output = resolve(base, output);
+    }
+    const set = new Set();
+    const useCache = options.cache !== false;
+    const debug = options.debug === true;
+    const verbose = options.verbose === true;
+    await foreachGLTF(input, async (file) => {
+        if (!existsSync(file)) {
+            // Safeguard if we run compression in a souce directory that already has progressive assets
+            // Here it can happen that a LOD file is generated with the same GUID at the moment 
+            // And then later removed because simplification is too small.
+            // See https://linear.app/needle/issue/NE-5518
+            if (isLOD(file)) {
+                logger.warn(`WARN: Ignore previously found file that does not exist anymore: \"${file}\"`);
+                return;
+            }
+            else {
+                logger.error(`ERR: Previously found file is now missing: \"${file}\"`);
+                process.exit(ERROR_CODES.LOST_FILE);
+            }
+        }
+        logger.info("→ Transform " + file);
+        const progressive_results = new Array();
+        if (options.progressive === true) {
+            if (await make_progressive({
+                path: file,
+                outpath: output,
+                results: progressive_results,
+                config,
+                debug,
+                verbose,
+                useCache,
+            }).catch(err => {
+                logger.error(err);
+                process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
+            }) === false) {
+                process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
+            }
+            if (verbose) {
+                console.log("Progressive results:\n", progressive_results);
+            }
+        }
+        if (options.compress === true) {
+            const opts = {
+                config,
+                stats,
+                packedFiles: set,
+                debug,
+                verbose,
+                useCache,
+                logger: logger,
+            };
+            // If we have produced progressive assets then the array already contains the input
+            if (progressive_results.length > 0) {
+                for (const res of progressive_results) {
+                    if (await packGLTF(res, res, opts) === false) {
+                        process.exit(ERROR_CODES.PACKING_FAILED);
+                    }
+                }
+            }
+            // If we have not produced progressive assets then we just need to compress the input
+            // We can not just check if the file is in the list of results because the output path might have changed the name
+            else {
+                if (await packGLTF(file, output, opts) === false) {
+                    process.exit(ERROR_CODES.PACKING_FAILED);
+                }
+            }
+        }
+    });
+    printStats(stats, logger, config);
+    limitCacheSize();
+});
+program.run().catch(err => {
+    // Since we still invoke this package with paths when running npm scripts we need to ignore this error
+    // We might want to move this code here into an extra package.
+    if (err.message.includes("Unknown command") && !err.message.includes("Did you mean")) {
+        // ignore
+        return;
+    }
+    throw err;
+});

package/dist/config/index.d.ts

@@ -0,0 +1,53 @@
+import { ILogger } from "@gltf-transform/core";
+declare type Usecase = "default" | "product" | "world";
+export declare const UsecaseOptions: string[];
+/**
+ * The config object contains information about how a file should be processed.
+ */
+export type Config = {
+    /** High-level configuration */
+    usecase: Usecase;
+    textures?: {
+        lods: boolean;
+    };
+    meshes?: {
+        lods: boolean;
+    };
+};
+/**
+ * Create a config object with the given usecase.
+ */
+export declare function createConfig(usecase: string): {
+    usecase: Usecase;
+};
+/**
+ * Get the config object either provided by a CLI path or a needle.config.json file in the parent directory.
+ *
+ * @example CLI argument where the path points to a json file containing the config as `Config`
+ * ```sh
+ * node script.js --config path/to/my-optimization-config.json
+ * ```
+ *
+ * @example needle.config.json - the usecase can be provided as a string
+ * ```json
+ * {
+ *    "optimization": "product"
+ * }
+ * ```
+ *
+ * @example needle.config.json - the whole config object can be provided
+ * ```json
+ * {
+ *   "optimization": {
+ *      "usecase": "product"
+ *   }
+ * }
+ * ```
+ *
+ *
+ */
+export declare function getConfig(configPath?: string, fallbackSearchDirectory?: string, opts?: {
+    logger: ILogger;
+}): Config;
+export declare function searchConfig(directory: string, logger: ILogger): Config | null;
+export {};