@needle-tools/gltf-build-pipeline 2.2.1-alpha.7 → 2.3.0-alpha

package/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.3.0-alpha] - 2023-06-22
+- add: flatten and join for `product` usecase
+
+## [2.2.1-alpha.8] - 2023-06-22
+- add: CLI <output> support 
+- fix: register webp extension for valid glTF files. [issue 5122](https://linear.app/needle/issue/NE-5122)
+- fix: issue where VRM material would be pruned
+- change: VRM use meshopt compression where any blendshapes are found
+
 ## [2.2.1-alpha.7] - 2023-06-19
 - fix: WARN if image validation can not run
 - change: compress lightmap textures with webp

package/dist/cli/index.js

@@ -6,9 +6,8 @@ import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
-import { statSync } from 'fs';
-import { foreachGLTF, isDirectory } from '../utils/fileutils.js';
-import path, { dirname } from 'path';
+import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
+import path, { resolve } from 'path';
 import { ERROR_CODES } from '../constants.js';
 // For testing / dev you can run `npm link` in the package directory
 // For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
@@ -57,7 +56,7 @@ This will produce multiple versions of the input file, each with a different lev
 Each version will be compressed and written to the output directory.
 `)
     .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
-    // .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
+    .argument('<output>', "The output file or directory. It can be an absolute path or a relative path to the input file. If not provided, the input file will be overwritten.", { default: null })
     .option("--compress", `When enabled files will be compressed`, { validator: program.BOOLEAN, default: true })
     .option("--progressive", `When enabled files will processed to be progressively loaded`, { validator: program.BOOLEAN, default: true })
     .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
@@ -78,8 +77,8 @@ Each version will be compressed and written to the output directory.
         totalFileSizeInMB: 0,
         totalFileSizeInMBBefore: 0,
     };
-    const file = args.input.toString();
-    let config = getConfig(options.config?.toString(), file, {
+    const input = args.input.toString();
+    let config = getConfig(options.config?.toString(), input, {
         logger
     });
     if (options.usecase) {
@@ -92,18 +91,29 @@ Each version will be compressed and written to the output directory.
             config.usecase = options.usecase.toString();
         }
     }
-    const output = options.output?.toString() || file;
-    const outputIsDirectory = statSync(output).isDirectory();
-    const outDir = outputIsDirectory ? output : dirname(output);
+    let output = args.output?.toString() || input;
+    const inputIsDirectory = isDirectory(input);
+    const outputIsDirectory = isDirectory(output);
+    if (inputIsDirectory && !outputIsDirectory) {
+        logger.error("When <input> is a directory then <output> must also be a directory");
+        process.exit(ERROR_CODES.INVALID_ARGS);
+    }
+    // Handle the output bath being relative to the input path
+    if (path.isAbsolute(output) === false) {
+        const base = ensureIsDirectory(input);
+        output = resolve(base, output);
+    }
     const set = new Set();
     const useCache = options.cache !== false;
     const debug = options.debug === true;
-    await foreachGLTF(file, async (file) => {
+    const verbose = options.verbose === true;
+    await foreachGLTF(input, async (file) => {
         logger.info("→ Transform " + file);
         const progressive_results = new Array();
         if (options.progressive === true) {
             if (await make_progressive({
                 path: file,
+                outpath: output,
                 results: progressive_results,
                 config,
                 debug,
@@ -121,16 +131,24 @@ Each version will be compressed and written to the output directory.
                 stats,
                 packedFiles: set,
                 debug,
+                verbose,
                 useCache,
                 logger: logger,
             };
-            for (const res of progressive_results) {
-                if (await packGLTF(res, res, opts) === false) {
-                    process.exit(ERROR_CODES.PACKING_FAILED);
+            // If we have produced progressive assets then the array already contains the input
+            if (progressive_results.length > 0) {
+                for (const res of progressive_results) {
+                    if (await packGLTF(res, res, opts) === false) {
+                        process.exit(ERROR_CODES.PACKING_FAILED);
+                    }
                 }
             }
-            if (await packGLTF(file, file, opts) === false) {
-                process.exit(ERROR_CODES.PACKING_FAILED);
+            // If we have not produced progressive assets then we just need to compress the input
+            // We can not just check if the file is in the list of results because the output path might have changed the name
+            else {
+                if (await packGLTF(file, output, opts) === false) {
+                    process.exit(ERROR_CODES.PACKING_FAILED);
+                }
             }
         }
     });

package/dist/constants.d.ts

@@ -2,4 +2,5 @@ export declare const ERROR_CODES: {
     UNKNOWN_ERROR: number;
     PACKING_FAILED: number;
     PROGRESSIVE_FAILED: number;
+    INVALID_ARGS: number;
 };

package/dist/constants.js

@@ -2,4 +2,5 @@ export const ERROR_CODES = {
     UNKNOWN_ERROR: 1,
     PACKING_FAILED: 2,
     PROGRESSIVE_FAILED: 3,
+    INVALID_ARGS: 4,
 };

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -524,6 +524,7 @@ class JsonPointerHandler {
     map = new Map();
     debug = false;
     breakPoint = false;
+    VRM_0_MaterialIndexRegex = new RegExp(/materialProperties\/(?<material_index>\d+)\//g);
     constructor(document, id) {
         this.document = document;
         this.id = id;
@@ -671,7 +672,7 @@ class JsonPointerHandler {
                 // key texture is matching for example for VRM meta/texture (AvatarSample_A)
                 key === "texture"
                     // VRM 0.0 material properties
-                    || fullPath.match(/materialProperties\/\d+\/textureProperties/g)
+                    || fullPath.match(/materialProperties\/(?<material_index>\d+)\/textureProperties/g)
                     // VRM 1.0 meta
                     || fullPath.match(/meta\/thumbnailImage/g)) {
                     const index = value;
@@ -705,6 +706,23 @@ class JsonPointerHandler {
                         this.map.set(dictKey, pointer);
                         prop.setReference(dictKey, object, { channels: TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A });
                     }
+                    const materialMatch = this.VRM_0_MaterialIndexRegex.exec(fullPath);
+                    if (materialMatch) {
+                        const materialIndex = parseInt(materialMatch.groups.material_index);
+                        const material = context.materials[materialIndex];
+                        prop.setReference(`/materials/${materialIndex}`, material);
+                        // const res = new MaterialIndexResolver(this.document, material, context.jsonDoc.json.materials![materialIndex]);
+                        // res.debug = this.debug;
+                        // res.init(context);
+                        // const dictKey = `/materials/${materialIndex}`
+                        // let pointer = this.map.get(dictKey);
+                        // if (!pointer) {
+                        //     pointer = new JsonPointer(null, res, value);
+                        // }
+                        // pointer.debug = this.debug;
+                        // pointer.register(obj, key, fullPath);
+                        // this.map.set(dictKey, pointer);
+                    }
                 }
                 break;
             default:

package/dist/scripts/pack-gltf.js

@@ -4,10 +4,10 @@ import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
 import draco3d from 'draco3dgltf';
 import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
-import { dedup, prune, resample } from '@gltf-transform/functions';
+import { dedup, flatten, join, prune, resample } from '@gltf-transform/functions';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, } from '../extensions/index.js';
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
-import { writeNodeIO } from "../utils/index.js";
+import { getOutputPath, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
 /** Compress a glTF file
  */
@@ -60,7 +60,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         });
         const document = await io.read(inputFile);
         /** @ts-ignore */
-        document.getLogger().verbosity = options.debug || options.verbose ? Verbosity.DEBUG : Verbosity.WARN;
+        document.getLogger().verbosity = options.verbose ? Verbosity.DEBUG : options.debug ? Verbosity.INFO : Verbosity.WARN;
         let totalGPUTextureMemory = 0;
         /** @type {Array<import('@gltf-transform/core').Transform>} */
         const transforms = [
@@ -91,8 +91,16 @@ export async function packGLTF(inputFile, outputFile, options) {
             }));
         }
         await document.transform(...transforms);
+        const isVRM = inputFile.toLowerCase().endsWith(".vrm");
+        if (!isVRM && config?.usecase === "product") {
+            await document.transform(flatten(), join({ keepNamed: false }));
+        }
         if (!outputFile)
             outputFile = inputFile;
+        else {
+            outputFile = getOutputPath(inputFile, outputFile);
+            logger.info(`⇄ Writing to ${outputFile}`);
+        }
         await writeNodeIO(io, outputFile, document);
         if (useCache && cacheKey != undefined && existsSync(outputFile))
             addToCache(cacheKey, readFileSync(outputFile));

package/dist/transforms/needle_mesh_transform.js

@@ -33,9 +33,12 @@ async function applyMeshCompression(document, transform_context, context) {
     }
     // Meshop is better at re-using vertex attributes
     if (context.file.endsWith(".vrm")) {
-        document.getLogger().info(`[${NAME}]: VRM → will apply meshopt mesh compression`);
-        const transform = meshopt({ encoder: MeshoptEncoder });
-        return transform(document, transform_context);
+        const hasBlendshapes = document.getRoot().listMeshes().some((mesh) => mesh.listPrimitives().some((prim) => prim.listTargets().length > 0));
+        if (hasBlendshapes) {
+            document.getLogger().info(`[${NAME}]: VRM → will apply meshopt mesh compression`);
+            const transform = meshopt({ encoder: MeshoptEncoder });
+            return transform(document, transform_context);
+        }
     }
     // by default we still want to run draco if the root doesnt have a compression model configured
     document.getLogger().info(`[${NAME}]: → will apply draco mesh compression (default)`);

package/dist/transforms/needle_progressive.d.ts

@@ -2,6 +2,8 @@ import { ILogger, Texture, Transform, vec2 } from '@gltf-transform/core';
 import { Config } from '../config/index.js';
 declare interface ProgressiveOptions {
     path: string | Array<string>;
+    /** Directory to output progressive assets to */
+    outpath?: string;
     results: Array<string>;
     config: Config;
     debug?: boolean;

package/dist/transforms/needle_progressive.js

@@ -1,13 +1,13 @@
 import { BufferUtils, Document, NodeIO, PropertyType, TextureChannel, Verbosity } from '@gltf-transform/core';
-import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureChannels, listTextureSlots, sparse } from '@gltf-transform/functions';
+import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, flatten, join, listTextureChannels, listTextureSlots, sparse } from '@gltf-transform/functions';
 import draco3d from 'draco3dgltf';
 import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive } from "../extensions/NEEDLE_progressive/index.js";
 import { createOpaqueExtension, currentGenerator, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
 import path, { basename, resolve } from 'path';
-import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
+import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
 import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
-import { detectTextureCompressionMode, foreachGLTF, generateGuid, isLightmap, setLightmapName, validateImageMimeType, writeNodeIO } from '../utils/index.js';
+import { detectTextureCompressionMode, ensureIsDirectory, foreachGLTF, generateGuid, getOutputPath, isLightmap, setLightmapName, validateImageMimeType, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive } from './needle_mesh_transform.js';
@@ -77,7 +77,7 @@ async function onProcess(file, opts) {
                 transforms.push(make_progressive_textures(file, { size: [smallesTextureSize, smallesTextureSize] }, opts));
             }
             // TODO: progressive meshes aren't supported yet for VRMs
-            const isVRM = file.endsWith(".vrm");
+            const isVRM = file.toLowerCase().endsWith(".vrm");
             if (!isVRM && opts.config?.meshes?.lods !== false) {
                 transforms.push(make_progressive_meshes(file, {}, opts));
             }
@@ -85,8 +85,12 @@ async function onProcess(file, opts) {
                 logger.warn("WARN: Progressive meshes are not supported for VRM files yet");
             transforms.push(sparse());
             logger.info("→ Transforming: " + file);
+            if (!isVRM && opts.config?.usecase === "product") {
+                await document.transform(flatten(), join({ keepNamed: false }));
+            }
             await document.transform(...transforms);
-            const filename = file;
+            const filename = getOutputPath(file, opts.outpath);
+            ;
             await writeNodeIO(io, filename, document);
             opts.results.push(filename);
             logger.info("← Transforming done: " + filename);
@@ -111,7 +115,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
     const startTime = Date.now();
     const filename = basename(filePath);
     const options = { ...TEXTURE_RESIZE_DEFAULTS, ..._options };
-    const directory = path.dirname(filePath);
+    let directory = path.dirname(filePath);
+    if (opts.outpath) {
+        directory = ensureIsDirectory(opts.outpath);
+    }
     // const fileNameWithoutExtension = path.basename(filePath, path.extname(filePath));
     const textureLodBasePath = directory + "/";
     return createTransform(NAME, async (doc) => {
@@ -367,7 +374,10 @@ async function createTextureLod(args, opts) {
     const ext = ".glb";
     const imageName = "image_" + level + "_" + guid;
     const filename = imageName + ext;
-    const outName = basePath + filename;
+    const outName = path.join(basePath, filename);
+    if (!existsSync(basePath)) {
+        mkdirSync(basePath, { recursive: true });
+    }
     logger.info(`Write texture LOD ${level} to \"${filename}\"`);
     newIo.write(outName, newDoc);
     return {
@@ -480,7 +490,10 @@ async function resizeTexture(options, texture, logger, uri, slots) {
     };
 }
 export function make_progressive_meshes(filePath, _options, opts) {
-    const directory = path.dirname(filePath);
+    let directory = path.dirname(filePath);
+    if (opts.outpath) {
+        directory = ensureIsDirectory(opts.outpath);
+    }
     // const fileNameWithoutExtension = path.basename(filePath, path.extname(filePath));
     const outputDirectory = directory + "/";
     return createTransform("PROGRESSIVE_MESHES", async (doc, _context) => {

package/dist/transforms/needle_toktx.js

@@ -2,7 +2,7 @@ import { BufferUtils, FileUtils, ImageUtils, uuid } from "@gltf-transform/core";
 import { listTextureSlots, getTextureChannelMask, fitPowerOfTwo, TextureResizeFilter } from "@gltf-transform/functions";
 import { KHRTextureBasisu } from "@gltf-transform/extensions";
 import { checkKTXSoftware, createParams, KTX_SOFTWARE_VERSION_MIN, Mode } from "./toktx.js";
-import { existsSync, mkdirSync, rmdirSync } from "fs";
+import { existsSync, mkdirSync, rmSync } from "fs";
 import { formatBytes, spawn, waitExit } from './util.js';
 import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "./toktx_types.js";
 import fs from 'fs/promises';
@@ -233,7 +233,7 @@ export class NeedleToktxContext {
             this.basisuExtension?.dispose();
         }
         if (this.cacheDirectory && existsSync(this.cacheDirectory)) {
-            rmdirSync(this.cacheDirectory, { recursive: true });
+            rmSync(this.cacheDirectory, { recursive: true });
         }
     }
 }

package/dist/transforms/needle_webp.d.ts

@@ -6,6 +6,7 @@ export interface NeedleWebPOptions {
 export declare class NeedleWebPContext implements INeedleTextureTransformContext {
     options: NeedleWebPOptions;
     logger: ILogger;
+    private compressed;
     constructor(options: NeedleWebPOptions);
     prepare(document: Document): Promise<void>;
     process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality">, context: NeedleTransformContext): Promise<boolean>;

package/dist/transforms/needle_webp.js

@@ -1,12 +1,15 @@
 import sharp from 'sharp';
 import { BufferUtils, HTTPUtils } from '@gltf-transform/core';
+import { EXTTextureWebP } from '@gltf-transform/extensions';
 export class NeedleWebPContext {
     options;
     logger;
+    compressed = 0;
     constructor(options) {
         this.options = options;
     }
     async prepare(document) {
+        this.compressed = 0;
         this.logger = document.getLogger();
     }
     async process(index, texture, settings, context) {
@@ -30,8 +33,13 @@ export class NeedleWebPContext {
             .setMimeType('image/webp')
             .setImage(BufferUtils.toView(buffer))
             .setURI(HTTPUtils.basename(texture.getURI()) + '.webp');
+        this.compressed += 1;
         return true;
     }
     async finalize(document) {
+        if (this.compressed > 0) {
+            const webpExtension = document.createExtension(EXTTextureWebP);
+            webpExtension.setRequired(true);
+        }
     }
 }

package/dist/utils/fileutils.d.ts

@@ -1,6 +1,19 @@
+/**
+ * Get the output path for a file based on the output directory or file.
+ * If outDirOrFile is a filepath, it will be returned as is.
+ * If outDirOrFile is a directory, the file will be placed in that directory.
+ */
+export declare function getOutputPath(filepath: string, outDirOrFile: string | undefined | null, opts?: {
+    create?: boolean;
+}): string;
 export declare function getFileName(filepath: string): string;
 export declare function getDirName(filepath: string): string;
 export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
 export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;
 export declare function replaceExtension(filepath: string, newExtension: string): string;
 export declare function isDirectory(path: string): boolean;
+/**
+ * Ensure that a path is a directory.
+ * If the directory does not exist, it will be created.
+ */
+export declare function ensureIsDirectory(path: string): string;

package/dist/utils/fileutils.js

@@ -1,5 +1,25 @@
-import { existsSync, statSync, readdirSync, readFileSync } from 'fs';
-import { isAbsolute } from 'path';
+import { existsSync, statSync, readdirSync, readFileSync, mkdirSync } from 'fs';
+import { dirname, isAbsolute, join } from 'path';
+/**
+ * Get the output path for a file based on the output directory or file.
+ * If outDirOrFile is a filepath, it will be returned as is.
+ * If outDirOrFile is a directory, the file will be placed in that directory.
+ */
+export function getOutputPath(filepath, outDirOrFile, opts) {
+    if (!outDirOrFile) {
+        return filepath;
+    }
+    if (!isDirectory(outDirOrFile)) {
+        return outDirOrFile;
+    }
+    const filename = getFileName(filepath);
+    // create the directory if it doesn't exist (default behavior)
+    if (opts?.create != false && !existsSync(outDirOrFile)) {
+        mkdirSync(outDirOrFile, { recursive: true });
+    }
+    const outpath = join(outDirOrFile, filename);
+    return outpath;
+}
 export function getFileName(filepath) {
     const parts = filepath.replaceAll("\\", "/").split("/");
     return parts[parts.length - 1];
@@ -139,3 +159,16 @@ export function isDirectory(path) {
     const hasExt = path.replaceAll("\\", "/").split("/").pop()?.includes(".");
     return !hasExt;
 }
+/**
+ * Ensure that a path is a directory.
+ * If the directory does not exist, it will be created.
+ */
+export function ensureIsDirectory(path) {
+    if (!isDirectory(path)) {
+        path = dirname(path);
+    }
+    if (!existsSync(path)) {
+        mkdirSync(path, { recursive: true });
+    }
+    return path;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.2.1-alpha.7",
+  "version": "2.3.0-alpha",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",