@needle-tools/gltf-build-pipeline 1.5.0-alpha.8 → 1.6.0-alpha

package/CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.6.0-alpha] - 2023-05-27
+- add: `gltf-build-pipeline` commandline support with `transform` and `compress` (pack-gltf) options.
+
+## [1.5.0-alpha.12] - 2023-05-20
+- add: pack-gltf now also runs `resample` for animations
+- fix: nullreference error in accessing opaque debug options
+
+## [1.5.0-alpha.10] - 2023-05-17
+- add: `generateLods` option to progressive texture settings
+
+## [1.5.0-alpha.9] - 2023-05-16
+- add: start of VRM 1.0 support. Not implemented is the `VRMC_materials_mtoon` extension 
+
 ## [1.5.0-alpha.8] - 2023-05-16
 - fix: issue where updating ktx compressed texture extension also flattened the path
 

package/dist/cache/cache.d.ts

@@ -1,9 +1,10 @@
 import { ExtensibleProperty } from "@gltf-transform/core";
+import { Logger } from "@donmccurdy/caporal";
 /**
  * Clear the cache.
  * This will remove all files in the cache directory.
  */
-export declare function clearCache(): void;
+export declare function clearCache(logger?: Logger): void;
 /**
  * Ensure that the cache size does not exceed the given maximum size in MB.
  * @param maxSizeMB The maximum size in MB.

package/dist/cache/cache.js

@@ -23,10 +23,13 @@ const buildPipelineVersion = getVersion(false);
  * Clear the cache.
  * This will remove all files in the cache directory.
  */
-export function clearCache() {
+export function clearCache(logger) {
     const dir = getCacheDirectory(true);
     fs.rmSync(dir, { recursive: true });
-    console.log(`Cleared cache at ${dir}`);
+    if (!logger)
+        console.log(`Cleared cache at ${dir}`);
+    else
+        logger.info(`Cleared cache at ${dir}`);
 }
 /**
  * Ensure that the cache size does not exceed the given maximum size in MB.
@@ -223,6 +226,9 @@ function hashObject(obj) {
         const buffer = obj.getArray();
         return hashBuffer(buffer);
     }
+    else if (obj == null) {
+        return 0;
+    }
     const str = JSON.stringify(obj);
     return hashString(str);
 }

package/dist/cli/index.d.ts

@@ -0,0 +1,2 @@
+#! /usr/bin/env node
+export {};

package/dist/cli/index.js

@@ -0,0 +1,102 @@
+#! /usr/bin/env node
+import { program } from '@donmccurdy/caporal';
+import { packGLTF } from "../scripts/index.js";
+import { printStats } from '../utils/stats.js';
+import { getVersion } from '../utils/version.js';
+import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
+import { make_progressive } from '../transforms/needle_progressive.js';
+import { clearCache } from '../cache/cache.js';
+// For testing / dev you can run `npm link` in the package directory
+// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
+program
+    .command("version", "Print current verison")
+    .action(async ({ logger }) => {
+    logger.info(getVersion());
+})
+    .command("clear-caches", "Clear all caches")
+    .help(`
+Clear all caches.
+`)
+    .action(async ({ logger }) => {
+    logger.info("Clearing caches");
+    clearCache(logger);
+})
+    .command("compress", "Compress a glTF file")
+    .help(`
+Compress a single glTF file. The input file is compressed and written to the output file.  
+`)
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed")
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Options are ${UsecaseOptions.join(", ")}`, { validator: program.STRING })
+    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
+    .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
+    .action(async ({ logger, args, options }) => {
+    logger.info("Transforming glTF file: " + args.input);
+    const stats = {
+        startTime: Date.now(),
+        totalGPUMemoryInMB: 0,
+        totalFilesProcessed: 0,
+        totalFileSizeInMB: 0,
+        totalFileSizeInMBBefore: 0,
+    };
+    let config = getConfig(options.config?.toString());
+    if (options.usecase) {
+        logger.info("Using usecase: " + options.usecase);
+        config = { ...config, ...createConfig(options.usecase.toString()) };
+    }
+    const file = args.input.toString();
+    const set = new Set();
+    await packGLTF(file, options.output?.toString() || file, {
+        config,
+        packedFiles: set,
+        stats,
+        debug: false,
+        useCache: true,
+    });
+    printStats(stats);
+})
+    .command("transform", "Transform a glTF file")
+    .help(`
+This will produce multiple versions of the input file, each with a different level of optimization.   
+Each version will be compressed and written to the output directory.
+`)
+    .argument('<input>', "The input glTF, GLB or VRM file that should be compressed")
+    .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Options are ${UsecaseOptions.join(", ")}`, { validator: program.STRING })
+    .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
+    // .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
+    .action(async ({ logger, args, options }) => {
+    logger.info("Transforming glTF file: " + args.input);
+    const stats = {
+        startTime: Date.now(),
+        totalGPUMemoryInMB: 0,
+        totalFilesProcessed: 0,
+        totalFileSizeInMB: 0,
+        totalFileSizeInMBBefore: 0,
+    };
+    let config = getConfig(options.config?.toString());
+    if (options.usecase) {
+        logger.info("Using usecase: " + options.usecase);
+        config = { ...config, ...createConfig(options.usecase.toString()) };
+    }
+    const file = args.input.toString();
+    const progressive_results = new Array();
+    await make_progressive({
+        path: file,
+        results: progressive_results,
+        config,
+        debug: false,
+        useCache: true,
+    });
+    console.log("Progressive results: ", progressive_results);
+    const opts = {
+        config,
+        stats,
+        debug: false,
+        useCache: true,
+    };
+    for (const prog of progressive_results) {
+        await packGLTF(prog, prog, opts);
+    }
+    await packGLTF(file, file, opts);
+    printStats(stats);
+});
+program.run();

package/dist/config/index.d.ts

@@ -0,0 +1,43 @@
+declare type Usecase = "unknown" | "product" | "game" | "metaverse";
+export declare const UsecaseOptions: string[];
+/**
+ * The config object contains information about how a file should be processed.
+ */
+export type Config = {
+    /** High-level configuration */
+    usecase: Usecase;
+};
+/**
+ * Create a config object with the given usecase.
+ */
+export declare function createConfig(usecase: string): {
+    usecase: Usecase;
+};
+/**
+ * Get the config object either provided by a CLI path or a needle.config.json file in the parent directory.
+ *
+ * @example CLI argument where the path points to a json file containing the config as `Config`
+ * ```sh
+ * node script.js --config path/to/my-optimization-config.json
+ * ```
+ *
+ * @example needle.config.json - the usecase can be provided as a string
+ * ```json
+ * {
+ *    "optimization": "product"
+ * }
+ * ```
+ *
+ * @example needle.config.json - the whole config object can be provided
+ * ```json
+ * {
+ *   "optimization": {
+ *      "usecase": "product"
+ *   }
+ * }
+ * ```
+ *
+ *
+ */
+export declare function getConfig(path?: string): Config;
+export {};

package/dist/config/index.js

@@ -0,0 +1,125 @@
+import { existsSync, readFileSync } from "fs";
+export const UsecaseOptions = ["product", "game", "metaverse"];
+const defaultConfig = {
+    usecase: "unknown",
+};
+/**
+ * Create a config object with the given usecase.
+ */
+export function createConfig(usecase) {
+    if (!UsecaseOptions.includes(usecase)) {
+        throw new Error(`Invalid usecase: \"${usecase}\". Valid usecases: ${UsecaseOptions.join(", ")}`);
+    }
+    return {
+        usecase: usecase
+    };
+}
+/**
+ * Get the config object either provided by a CLI path or a needle.config.json file in the parent directory.
+ *
+ * @example CLI argument where the path points to a json file containing the config as `Config`
+ * ```sh
+ * node script.js --config path/to/my-optimization-config.json
+ * ```
+ *
+ * @example needle.config.json - the usecase can be provided as a string
+ * ```json
+ * {
+ *    "optimization": "product"
+ * }
+ * ```
+ *
+ * @example needle.config.json - the whole config object can be provided
+ * ```json
+ * {
+ *   "optimization": {
+ *      "usecase": "product"
+ *   }
+ * }
+ * ```
+ *
+ *
+ */
+export function getConfig(path) {
+    if (path != null) {
+        if (!existsSync(path)) {
+            throw new Error(`No config found at \"${path}\"`);
+        }
+        return resolveConfig(path, readFileSync(path, "utf8"));
+    }
+    if (_cachedConfig !== undefined) {
+        return _cachedConfig;
+    }
+    // Resolve the config once and cache it
+    const result = internalResolveConfig();
+    if (result) {
+        console.log("INFO: Using optimization config with usecase: " + result.usecase);
+        _cachedConfig = result;
+    }
+    else {
+        console.log("INFO: No optimization config found. Using default config.");
+        _cachedConfig = defaultConfig;
+    }
+    return _cachedConfig;
+}
+let _cachedConfig = undefined;
+function internalResolveConfig() {
+    // Check if a config path is passed as an argument
+    const args = process.argv.slice(2);
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === "--config") {
+            if (i + 1 < args.length) {
+                const configPath = args[i + 1];
+                if (existsSync(configPath)) {
+                    const content = readFileSync(configPath, "utf8");
+                    return resolveConfig(configPath, content);
+                }
+                throw new Error(`Config file not found at path configured by CLI --config argument: \"${configPath}\"`);
+            }
+        }
+    }
+    // Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
+    const workingDir = process.cwd();
+    let directory = workingDir.replaceAll("\\", "/");
+    while (directory.length > 0) {
+        const configPath = `${directory}/needle.config.json`;
+        if (existsSync(configPath)) {
+            const content = readFileSync(configPath, "utf8");
+            const config = resolveConfig(configPath, content);
+            if (config) {
+                return config;
+            }
+        }
+        const lastSlash = directory.lastIndexOf("/");
+        if (lastSlash < 0) {
+            break;
+        }
+        directory = directory.slice(0, lastSlash);
+    }
+    return null;
+}
+function resolveConfig(filename, content) {
+    // Handle config being part of `needle.config.json`
+    if (filename.endsWith("needle.config.json")) {
+        const config = JSON.parse(content);
+        if (config.optimization) {
+            console.log("INFO: Found optimization config at \"" + filename + "\".");
+            // If the optimization config is a string, it's the usecase
+            if (typeof config.optimization === "string") {
+                return createConfig(config.optimization);
+            }
+            return config.optimization;
+        }
+        else {
+            console.log("INFO: No optimization config found in needle.config.json. Using default config.");
+            return null;
+        }
+    }
+    // Handle config being the content of the file
+    else if (content.trimStart().startsWith("{")) {
+        return JSON.parse(content);
+    }
+    else {
+        throw new Error("Invalid config file format. Expected JSON object.");
+    }
+}

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts

@@ -1,9 +1,10 @@
 import { WriterContext, PropertyType } from '@gltf-transform/core';
 declare interface IExtensibleProperty {
 }
+type DebugBreakPoint = boolean | "pointer:read";
 export type Options = {
     debug?: boolean;
-    breakpoint?: boolean;
+    breakpoint?: DebugBreakPoint;
     write?: boolean;
 };
 export interface IOpaqueExtension {

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -19,13 +19,14 @@ export function createOpaqueExtension(name, types, opts) {
         // private roots: Array<OpaqueProperty> = [];
         jsonPointerHandler;
         read(context) {
-            if (opts.breakpoint === true)
+            if (opts?.breakpoint === true)
                 debugger;
             currentGenerator = context.jsonDoc.json.asset?.generator;
             if (debugLog)
                 console.log("READ " + EXTENSION_NAME);
             this.jsonPointerHandler = new JsonPointerHandler(this.document, EXTENSION_NAME);
             this.jsonPointerHandler.debug = debugLog;
+            this.jsonPointerHandler.breakPoint = opts?.breakpoint;
             for (const type of types) {
                 switch (type) {
                     case PropertyType.ACCESSOR:
@@ -224,7 +225,7 @@ export function createOpaqueExtension(name, types, opts) {
             }
         }
         write(context) {
-            if (opts.breakpoint === true)
+            if (opts?.breakpoint === true)
                 debugger;
             if (opts?.write === false) {
                 return this;
@@ -514,6 +515,7 @@ class JsonPointerHandler {
     document;
     map = new Map();
     debug = false;
+    breakPoint = false;
     constructor(document, id) {
         this.document = document;
         this.id = id;
@@ -524,7 +526,11 @@ class JsonPointerHandler {
         if (key !== undefined) {
             fullPath ??= "";
             fullPath += "/" + key;
+            if (this.breakPoint === "pointer:read") {
+                debugger;
+            }
         }
+        // if(this.debug && key === "colliders") debugger;
         switch (typeof value) {
             case 'object':
                 if (value !== null && value !== undefined) {
@@ -570,7 +576,7 @@ class JsonPointerHandler {
                         if (pointer) {
                             if (this.debug)
                                 console.log("> Register pointer", this.id + "/" + fullPath + " : " + value);
-                            pointer?.register(obj, key);
+                            pointer?.register(obj, key, fullPath);
                         }
                         else
                             console.warn("WARN: failed registering pointer", fullPath, value);
@@ -597,13 +603,15 @@ class JsonPointerHandler {
                                 pointer.debug = this.debug;
                                 this.map.set(value, pointer);
                             }
-                            pointer?.register(obj, key);
+                            pointer?.register(obj, key, fullPath);
                         }
                     }
                 }
                 break;
             case 'number':
-                if (key === "node" || key === "mesh") {
+                if (key === "node" || key === "mesh"
+                    // VRM 1.0 spring bone center https://github.com/vrm-c/vrm-specification/blob/master/specification/VRMC_springBone-1.0/schema/VRMC_springBone.spring.schema.json
+                    || fullPath.match(/springs\/\d+\/center/g)) {
                     const object = context.nodes[value];
                     if (object) {
                         const res = new NodeIndexResolver(this.document, object, context.jsonDoc.json.nodes[value]);
@@ -615,12 +623,17 @@ class JsonPointerHandler {
                             pointer = new JsonPointer(null, res, value);
                         }
                         pointer.debug = this.debug;
-                        pointer.register(obj, key);
+                        pointer.register(obj, key, fullPath);
                         this.map.set(dictKey, pointer);
                     }
                 }
-                // key texture is matching for example for VRM meta/texture (AvatarSample_A) and the regex is also for handling VRM material properties
-                else if (key === "texture" || fullPath.match(/materialProperties\/\d+?\/textureProperties/g)) {
+                else if (
+                // key texture is matching for example for VRM meta/texture (AvatarSample_A)
+                key === "texture"
+                    // VRM 0.0 material properties
+                    || fullPath.match(/materialProperties\/\d+\/textureProperties/g)
+                    // VRM 1.0 meta
+                    || fullPath.match(/meta\/thumbnailImage/g)) {
                     const index = value;
                     // e.g. in meta.texture for VRM avatars it seems like we can have negative values
                     if (index < 0) {
@@ -648,7 +661,7 @@ class JsonPointerHandler {
                             pointer = new JsonPointer(null, res, value);
                         }
                         pointer.debug = this.debug;
-                        pointer.register(obj, key);
+                        pointer.register(obj, key, fullPath);
                         this.map.set(dictKey, pointer);
                         if (ext) {
                             ext.setReference(dictKey, object);
@@ -656,6 +669,8 @@ class JsonPointerHandler {
                     }
                 }
                 break;
+            default:
+                break;
         }
     }
     resolveAndWrite(context) {
@@ -677,9 +692,11 @@ class JsonPointer {
     }
     _objects = [];
     _props = [];
-    register(obj, prop) {
+    _debugInfo = [];
+    register(obj, prop, fullpath) {
         this._objects.push(obj);
         this._props.push(prop);
+        this._debugInfo.push(fullpath);
     }
     newPointer;
     resolve(context) {
@@ -694,18 +711,19 @@ class JsonPointer {
                 this.newPointer = index;
             }
             if (this.debug)
-                console.log(`RESOLVED ${this.oldPointer} to → ${this.newPointer}`);
+                console.log(`RESOLVED pointer ${this.oldPointer} to → ${this.newPointer}`);
         }
         else {
-            console.log("ERR: failed to resolve pointer", this.pointerBase);
+            console.log("ERR: 🧨 FAILED to resolve pointer", this.pointerBase, ` used by ${this._debugInfo.join(", ")}`);
         }
     }
     write() {
         for (let i = 0; i < this._objects.length; i++) {
             const obj = this._objects[i];
             const prop = this._props[i];
-            if (this.debug)
-                console.log("< Resolved pointer", this.newPointer, "in", prop);
+            if (this.debug) {
+                console.log(`< Resolved pointer ${this._debugInfo[i]}: ${this.newPointer}`);
+            }
             obj[prop] = this.newPointer;
         }
     }

package/dist/extensions/NEEDLE_opaque/index.d.ts

@@ -1,3 +1,3 @@
-import { IOpaqueExtension } from "./NEEDLE_opaque.js";
+import { Extension } from "@gltf-transform/core";
 export * from "./NEEDLE_opaque.js";
-export declare const ALL_EXTENSIONS: IOpaqueExtension[];
+export declare const ALL_EXTENSIONS: (typeof Extension)[];

package/dist/extensions/NEEDLE_opaque/index.js

@@ -17,5 +17,12 @@ export const ALL_EXTENSIONS = [
     createOpaqueExtension("EXT_texture_exr", [PropertyType.TEXTURE], { ...opaqueOptions }),
     createOpaqueExtension("NEEDLE_editor", [PropertyType.NODE, PropertyType.MATERIAL], { ...opaqueOptions }),
     createOpaqueExtension("NEEDLE_gltf_dependencies", [PropertyType.ROOT], { ...opaqueOptions }),
+    // VRM 0.0
     createOpaqueExtension("VRM", [PropertyType.ROOT], { ...opaqueOptions }),
+    // VRM 1.0 https://vrm.dev/en/vrm1/changed/
+    createOpaqueExtension("VRMC_vrm", [PropertyType.ROOT], { ...opaqueOptions }),
+    createOpaqueExtension("VRMC_springBone", [PropertyType.ROOT], { ...opaqueOptions }),
+    createOpaqueExtension("VRMC_node_constraint", [PropertyType.NODE], { ...opaqueOptions }),
+    // TODO: mtoon materials don't work yet
+    // createOpaqueExtension("VRMC_materials_mtoon", [PropertyType.MATERIAL], { ...opaqueOptions, debug: true, breakpoint: "pointer:read" },),
 ];

package/dist/extensions/NEEDLE_progressive_mesh_settings/index.d.ts

@@ -1,7 +1,10 @@
 import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare interface IProgressiveMeshSettings {
-    guid: string;
+    /**
+     * When false no LODs will be generated for this mesh
+     */
     generateLods: boolean;
+    guid: string;
 }
 export declare class NEEDLE_progressive_mesh_settings extends Extension {
     static EXTENSION_NAME: string;

package/dist/extensions/NEEDLE_progressive_texture_settings/index.d.ts

@@ -1,5 +1,9 @@
 import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare interface IProgressiveTextureSettings {
+    /**
+     * When false the texture no progressive texture will be generated
+     */
+    generateLods?: boolean;
     maxSize: number;
     guid: string;
 }

package/dist/index.d.ts

@@ -1,4 +1,7 @@
+export * from "./config/index.js";
+export * from "./cli/index.js";
 export * from "./utils/index.js";
 export * from "./transforms/index.js";
 export * from "./extensions/index.js";
 export * from "./cache/index.js";
+export * from "./scripts/index.js";

package/dist/index.js

@@ -1,4 +1,7 @@
+export * from "./config/index.js";
+export * from "./cli/index.js";
 export * from "./utils/index.js";
 export * from "./transforms/index.js";
 export * from "./extensions/index.js";
 export * from "./cache/index.js";
+export * from "./scripts/index.js";

package/dist/scripts/index.d.ts

@@ -0,0 +1 @@
+export * from "./pack-gltf.js";

package/dist/scripts/index.js

@@ -0,0 +1 @@
+export * from "./pack-gltf.js";

package/dist/scripts/pack-gltf.d.ts

@@ -0,0 +1,12 @@
+import { Stats } from "../utils/index.js";
+import { Config } from '../config/index.js';
+export type PackGLTFOptions = {
+    debug: boolean;
+    useCache: boolean;
+    packedFiles?: Set<string>;
+    stats?: Stats;
+    config?: Config;
+};
+/** Compress a glTF file
+ */
+export declare function packGLTF(inputFile: string, outputFile: string, options: PackGLTFOptions): Promise<boolean>;