@needle-tools/gltf-build-pipeline 2.4.0-alpha.3 → 2.4.0-alpha.5

package/CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.4.0-beta.5] - 2023-09-06
+- fix: issue where mesh LODs would in some cases be compressed with "draco" while the root file would be compressed with "mesh-opt" if not explicitly specified in the root file. This would cause invalid rendering and randomly scaled LOD meshes. Now mesh LODs will always use the same mesh compression. [NE 5297](https://linear.app/needle/issue/NE-5297)
+- change: update gltf-transform dependency to 4.0.8
+
+## [2.4.0-beta] - 2023-08-13
+- fix: issue where a mesh lod is removed during second processing of a directory that contains existing mesh lods
+
 ## [2.4.0-alpha.3] - 2023-08-11
 - fix: issue where files would not correctly be overriden in subdirectories when no output directory was specified 
 

package/dist/cli/index.js

@@ -4,8 +4,9 @@ import { calculateStats, packGLTF } from "../scripts/index.js";
 import { printStats, writeFileStatsToFile } from '../utils/stats.js';
 import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
-import { make_progressive } from '../transforms/needle_progressive.js';
+import { isLOD, make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
+import { existsSync } from 'fs';
 import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
 import path, { resolve } from 'path';
 import { ERROR_CODES } from '../constants.js';
@@ -112,6 +113,20 @@ Each version will be compressed and written to the output directory.
     const debug = options.debug === true;
     const verbose = options.verbose === true;
     await foreachGLTF(input, async (file) => {
+        if (!existsSync(file)) {
+            // Safeguard if we run compression in a souce directory that already has progressive assets
+            // Here it can happen that a LOD file is generated with the same GUID at the moment 
+            // And then later removed because simplification is too small.
+            // See https://linear.app/needle/issue/NE-5518
+            if (isLOD(file)) {
+                logger.warn(`WARN: Ignore previously found file that does not exist anymore: \"${file}\"`);
+                return;
+            }
+            else {
+                logger.error(`ERR: Previously found file is now missing: \"${file}\"`);
+                process.exit(ERROR_CODES.LOST_FILE);
+            }
+        }
         logger.info("→ Transform " + file);
         const progressive_results = new Array();
         if (options.progressive === true) {
@@ -121,6 +136,7 @@ Each version will be compressed and written to the output directory.
                 results: progressive_results,
                 config,
                 debug,
+                verbose,
                 useCache,
             }).catch(err => {
                 logger.error(err);
@@ -128,6 +144,9 @@ Each version will be compressed and written to the output directory.
             }) === false) {
                 process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
             }
+            if (verbose) {
+                console.log("Progressive results:\n", progressive_results);
+            }
         }
         if (options.compress === true) {
             const opts = {

package/dist/constants.d.ts

@@ -3,4 +3,5 @@ export declare const ERROR_CODES: {
     PACKING_FAILED: number;
     PROGRESSIVE_FAILED: number;
     INVALID_ARGS: number;
+    LOST_FILE: number;
 };

package/dist/constants.js

@@ -3,4 +3,5 @@ export const ERROR_CODES = {
     PACKING_FAILED: 2,
     PROGRESSIVE_FAILED: 3,
     INVALID_ARGS: 4,
+    LOST_FILE: 5,
 };

package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.d.ts

@@ -1,6 +1,7 @@
 import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare class NEEDLE_mesh_compression_model_root {
     compression: "none" | "draco" | "meshopt";
+    constructor(compression: "none" | "draco" | "meshopt");
 }
 export declare class NEEDLE_mesh_compression_model {
     useSimplifier: boolean;

package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js

@@ -3,6 +3,9 @@ const EXTENSION_NAME = "NEEDLE_mesh_compression";
 // TODO: this extension can now be on the root and on primitives containing different contents
 export class NEEDLE_mesh_compression_model_root {
     compression;
+    constructor(compression) {
+        this.compression = compression;
+    }
 }
 export class NEEDLE_mesh_compression_model {
     useSimplifier;

package/dist/scripts/pack-gltf.js

@@ -17,7 +17,7 @@ export async function packGLTF(inputFile, outputFile, options) {
     }
     const { debug, stats, packedFiles, useCache, logger } = options;
     if (!existsSync(inputFile)) {
-        logger.error("WARN: can not pack, file does not exist: " + inputFile);
+        logger.error(`WARN: can not pack, file does not exist: \"${inputFile}\"`);
         return false;
     }
     // check if it's in the cache. 

package/dist/transforms/needle_mesh_transform.d.ts

@@ -1,10 +1,15 @@
 import { Transform, Document, Primitive, Mesh } from "@gltf-transform/core";
 import { SimplifyOptions, WeldOptions } from "@gltf-transform/functions";
+import { NEEDLE_mesh_compression_model_root } from "../extensions/index.js";
 import { Config } from "../config/index.js";
 declare type MeshTransformContext = {
     file: string;
     config: Config;
 };
+/**
+ * Which compression type for a model should be used. This should be respected when creating LOD meshes and the appropriate extension should be added to the LOD mesh so it receives the same mesh compression as the parent mesh
+ */
+export declare function determineMeshCompression(document: Document, filePath: string): NEEDLE_mesh_compression_model_root;
 export declare const needle_mesh_transform: (context: MeshTransformContext, simplify: Omit<SimplifyOptions, "simplifier">) => Transform;
 export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, index: number, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<Primitive>;
 export {};

package/dist/transforms/needle_mesh_transform.js

@@ -1,43 +1,27 @@
 import { Primitive } from "@gltf-transform/core";
 import { meshopt, simplifyPrimitive, SIMPLIFY_DEFAULTS, WELD_DEFAULTS, draco } from "@gltf-transform/functions";
 import { MeshoptEncoder, MeshoptSimplifier } from "meshoptimizer";
-import { NEEDLE_mesh_compression } from "../extensions/index.js";
+import { NEEDLE_mesh_compression, NEEDLE_mesh_compression_model_root } from "../extensions/index.js";
 import { weldPrimitive } from "../functions/weld.3.js";
 const NAME = 'needle_mesh_transform';
-export const needle_mesh_transform = function (context, simplify) {
-    return async (document, _context) => {
-        // TODO: needs to be moved into NEEDLE_progressive
-        await simplifierTransform(document, _context, simplify);
-        await applyMeshCompression(document, _context, context);
-    };
-};
-async function applyMeshCompression(document, transform_context, context) {
+/**
+ * Which compression type for a model should be used. This should be respected when creating LOD meshes and the appropriate extension should be added to the LOD mesh so it receives the same mesh compression as the parent mesh
+ */
+export function determineMeshCompression(document, filePath) {
     const rootExtension = document.getRoot().getExtension(NEEDLE_mesh_compression.EXTENSION_NAME);
     if (rootExtension) {
         const rootModel = rootExtension.model;
-        if (rootModel.compression) {
-            if (rootModel.compression === "none") {
-                document.getLogger().info(`[${NAME}]: NONE → will not apply mesh compression`);
-                return;
-            }
-            document.getLogger().info(`[${NAME}]: ${rootModel.compression.toUpperCase()} → will apply ${rootModel.compression} mesh compression`);
-            if (rootModel.compression === "draco") {
-                const transform = draco();
-                return transform(document, transform_context);
-            }
-            else if (rootModel.compression === "meshopt") {
-                const transform = meshopt({ encoder: MeshoptEncoder });
-                return transform(document, transform_context);
+        if (rootModel) {
+            if (!rootModel.compression) {
+                throw new Error(`[${NAME}]: Invalid mesh compression model extension on root node of ${filePath}`);
             }
+            return rootModel;
         }
     }
-    // Meshop is better at re-using vertex attributes
-    if (context.file.endsWith(".vrm")) {
+    if (filePath.endsWith(".vrm")) {
         const hasBlendshapes = document.getRoot().listMeshes().some((mesh) => mesh.listPrimitives().some((prim) => prim.listTargets().length > 0));
         if (hasBlendshapes) {
-            document.getLogger().info(`[${NAME}]: VRM → will apply meshopt mesh compression`);
-            const transform = meshopt({ encoder: MeshoptEncoder });
-            return transform(document, transform_context);
+            return new NEEDLE_mesh_compression_model_root("meshopt");
         }
     }
     // heuristic to detect if we should choose meshopt over draco
@@ -56,8 +40,7 @@ async function applyMeshCompression(document, transform_context, context) {
     // TODO: make this more robus with progressive loading: when progressive loading is enabled we need to run this check on the root mesh and enforce the same mesh compression for all children. Until we can do that we can not use the code for choosing compression based on blendshape size
     if (sizeOfBlendshapes > 0) {
         document.getLogger().info(`[${NAME}]: will apply meshopt mesh compression because the mesh contains blendshapes`);
-        const transform = meshopt({ encoder: MeshoptEncoder });
-        return transform(document, transform_context);
+        return new NEEDLE_mesh_compression_model_root("meshopt");
     }
     // consider meshopt if blendshape size is > 256 KB
     // const sizeThreshold = 256 * 1024;
@@ -72,6 +55,35 @@ async function applyMeshCompression(document, transform_context, context) {
     //         return transform(document, transform_context);
     //     }
     // }
+    return new NEEDLE_mesh_compression_model_root("draco");
+}
+export const needle_mesh_transform = function (context, simplify) {
+    return async (document, _context) => {
+        // TODO: needs to be moved into NEEDLE_progressive
+        await simplifierTransform(document, _context, simplify);
+        await applyMeshCompression(document, _context, context);
+    };
+};
+async function applyMeshCompression(document, transform_context, context) {
+    const compressionModel = determineMeshCompression(document, context.file); // document.getRoot().getExtension(NEEDLE_mesh_compression.EXTENSION_NAME) as NEEDLE_mesh_compression_property;
+    if (compressionModel) {
+        // const rootModel = rootExtension.model as NEEDLE_mesh_compression_model_root;
+        if (compressionModel.compression) {
+            if (compressionModel.compression === "none") {
+                document.getLogger().info(`[${NAME}]: NONE → will not apply mesh compression`);
+                return;
+            }
+            document.getLogger().info(`[${NAME}]: ${compressionModel.compression.toUpperCase()} → will apply ${compressionModel.compression} mesh compression`);
+            if (compressionModel.compression === "draco") {
+                const transform = draco();
+                return transform(document, transform_context);
+            }
+            else if (compressionModel.compression === "meshopt") {
+                const transform = meshopt({ encoder: MeshoptEncoder });
+                return transform(document, transform_context);
+            }
+        }
+    }
     // by default we still want to run draco if the root doesnt have a compression model configured
     document.getLogger().info(`[${NAME}]: → will apply draco mesh compression (default)`);
     return draco()(document, transform_context);

package/dist/transforms/needle_progressive.js

@@ -10,7 +10,7 @@ import sharp from "sharp";
 import { detectTextureCompressionMode, ensureIsDirectory, foreachGLTF, generateGuid, getOutputPath, isLightmap, setLightmapName, validateImageMimeType, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
-import { compressPrimitive } from './needle_mesh_transform.js';
+import { compressPrimitive, determineMeshCompression } from './needle_mesh_transform.js';
 import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
 export async function make_progressive(opts) {
     const file = opts.path;
@@ -65,7 +65,7 @@ async function onProcess(file, opts) {
         if (document) {
             const logger = document.getLogger();
             //@ts-ignore
-            logger.verbosity = opts.debug || opts.verbose ? Verbosity.DEBUG : Verbosity.WARN;
+            logger.verbosity = (opts.debug || opts.verbose) ? Verbosity.DEBUG : Verbosity.WARN;
             let smallesTextureSize = 128;
             if (opts.config.usecase === "product") {
                 smallesTextureSize = 512;
@@ -652,6 +652,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
                         }
                     }
                     if (shouldOmitLevel) {
+                        logger.debug(`[${NAME}] Omitting LOD ${level} for \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                         generatedLODInfos.splice(i, 1);
                         rmSync(fullPath);
                         i--;
@@ -906,12 +907,11 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
         density: density,
     }));
     // we need to compress the mesh with the same compression method as the original file
-    const root = doc.getRoot();
-    const compressionExt = root.getExtension(NEEDLE_mesh_compression.EXTENSION_NAME);
-    if (compressionExt) {
+    const model = determineMeshCompression(doc, fileName);
+    if (model) {
         const newExt = newDoc.createExtension(NEEDLE_mesh_compression);
-        const newProp = newExt.create(compressionExt.model);
-        logger.info(`[${NAME}] Pass compression format to mesh LOD ${level} → ${JSON.stringify(compressionExt.model)}`);
+        const newProp = newExt.create(model);
+        logger.info(`[${NAME}] Pass compression format to mesh LOD ${level} → ${JSON.stringify(model)}`);
         newDoc.getRoot().setExtension(NEEDLE_mesh_compression.EXTENSION_NAME, newProp);
     }
     await newDoc.transform(sparse());

package/dist/utils/fileutils.js

@@ -77,7 +77,7 @@ export async function foreachGLTF(args, callback) {
         }
     }
     if (context.files.length <= 0) {
-        console.log("WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory. This path may be relative to \"" + initDir + "\"");
+        console.log(`WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory.\nBase Directory: ${context.baseDirectory}\nPaths can be relative to \"${initDir}\"`);
     }
     else {
         for (const file of context.files) {
@@ -99,13 +99,13 @@ async function testEntry(entry, context) {
         // If no base directory is defined we cannot try to resolve the path
         if (context.baseDirectory === undefined) {
             console.log("WARN: Path does not exist: " + entry);
-            return;
+            return false;
         }
         // Try resolve potential relative path
         const newPath = context.baseDirectory + "/" + entry;
         if (existsSync(newPath) === false) {
             console.log(`INFO: Resolved path failed: \"${entry}\" -> ${newPath} (doesnt exist or is locked by another process)`);
-            return;
+            return false;
         }
         else {
             console.log(`INFO: Resolved path: ${entry} -> ${newPath}`);
@@ -122,7 +122,7 @@ async function testEntry(entry, context) {
                 console.log(`INFO: Resolved path: ${entry} -> ${newPath}`);
                 entry = newPath;
                 if (!existsSync(entry)) {
-                    return;
+                    return false;
                 }
             }
         }
@@ -135,7 +135,7 @@ async function testEntry(entry, context) {
         if (stat.isDirectory()) {
             // Don't traverse node_modules
             if (entry.endsWith("node_modules")) {
-                return;
+                return false;
             }
             const files = readdirSync(entry);
             for (const file of files) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.4.0-alpha.3",
+  "version": "2.4.0-alpha.5",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",
@@ -47,9 +47,9 @@
   },
   "dependencies": {
     "@donmccurdy/caporal": "~0.0.10",
-    "@gltf-transform/core": "^4.0.2",
-    "@gltf-transform/extensions": "^4.0.2",
-    "@gltf-transform/functions": "^4.0.2",
+    "@gltf-transform/core": "^4.0.8",
+    "@gltf-transform/extensions": "^4.0.8",
+    "@gltf-transform/functions": "^4.0.8",
     "command-exists": "^1.2.9",
     "command-line-args": "^5.2.1",
     "draco3dgltf": "^1.5.3",