@needle-tools/gltf-build-pipeline 2.7.0-alpha → 2.7.0-alpha.2

package/CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.7.0-alpha.2] - 2024-02-13
+- fix: processing for some very large meshes where hashing the meshes failed (for caching the results)
+
+## [2.7.0-alpha.1] - 2024-01-13
+- fix: handle LOD omitting and remove deleted files from progressive transform results list
+
 ## [2.7.0-alpha] - 2024-01-13
 - add: VRM mesh simplification support
 - fix: issue where the output directory was not created if it didn't exist yet 

package/dist/cache/cache.d.ts

@@ -44,7 +44,7 @@ export declare function tryGetMetaFromCache(key: string, ignoreVersion?: boolean
 export declare function getKey(property: ExtensibleProperty): string;
 export declare function getKeyWithHash(property: ExtensibleProperty): {
     key: string;
-    hash: number;
+    hash: string;
 };
 /**
  * Get the hash for a property or an object.
@@ -52,4 +52,4 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
  * @param level The current level of recursion.
  * @returns The hash as a number.
  */
-export declare function getHash(property: ExtensibleProperty | object, level?: number): number;
+export declare function getHash(property: ExtensibleProperty | object, level?: number): string;

package/dist/cache/cache.js

@@ -181,7 +181,7 @@ export function getKeyWithHash(property) {
  * @returns The hash as a number.
  */
 export function getHash(property, level = 0) {
-    let hash = 0;
+    let hash = "";
     if (property instanceof ExtensibleProperty) {
         const extensions = property.listExtensions();
         const extras = property.getExtras();
@@ -209,15 +209,24 @@ export function getHash(property, level = 0) {
             const attributes = property.listAttributes();
             const targets = property.listTargets();
             // TODO: hash buffer view if possible
-            const semantics = property.listSemantics().map(s => {
-                return { name: s, array: property.getAttribute(s).getArray() };
+            const semantics = property.listSemantics();
+            const semanticsNames = semantics.map(s => {
+                return {
+                    name: s,
+                };
             });
             hash += hashObject({
                 indices,
                 attributes,
                 targets,
-                semantics
+                semantics: semanticsNames
             });
+            // We can not pass the whole buffer into hashObject since very large meshes will cause errors when trying to stringify
+            // See https://linear.app/needle/issue/NE-6414
+            for (const sem of semantics) {
+                const arr = property.getAttribute(sem).getArray();
+                hash += hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength);
+            }
         }
     }
     else {
@@ -236,13 +245,20 @@ function getCacheDirectory(root) {
 function hashObject(obj) {
     if (obj instanceof Accessor) {
         const buffer = obj.getArray();
-        return hashBuffer(buffer);
+        return hashBuffer(buffer.buffer);
     }
     else if (obj == null) {
         return 0;
     }
-    const str = JSON.stringify(obj);
-    return hashString(str);
+    try {
+        const str = JSON.stringify(obj);
+        return hashString(str);
+    }
+    catch (err) {
+        console.error(err);
+        debugger;
+        throw err;
+    }
 }
 function hashBuffer(buffer, offset, length) {
     let hash = 0;

package/dist/transforms/needle_progressive.js

@@ -29,6 +29,19 @@ export async function make_progressive(opts) {
         if (!res)
             success = false;
     });
+    if (success) {
+        // Make sure all results exist
+        // It's possible that files were deleted while omitting LODs - we want to remove them from the results list
+        const resultsCount = opts.results.length;
+        for (let i = opts.results.length - 1; i >= 0; i--) {
+            if (!existsSync(opts.results[i])) {
+                opts.results.splice(i, 1);
+            }
+        }
+        const resultsChanged = resultsCount !== opts.results.length;
+        if (opts.debug)
+            console.log(`Progressive Transform: ${opts.results.length} files processed. ${resultsChanged ? "Some results were removed" : ""}`);
+    }
     return success;
 }
 function registerExtensions(io) {
@@ -384,12 +397,14 @@ async function createTextureLod(args, opts) {
     if (!existsSync(basePath)) {
         mkdirSync(basePath, { recursive: true });
     }
-    logger.info(`Write texture LOD ${level} to \"${filename}\"`);
+    logger.info(`[${NAME}] Write texture LOD ${level} to \"${filename}\"`);
     newIo.write(outName, newDoc);
+    logger.info(`[${NAME}] Generate hash for texture LOD ${level}`);
+    const hash = getHash(newTexture);
     return {
         outpath: outName,
         texture: newTexture,
-        hash: getHash(newTexture).toString(),
+        hash: hash,
     };
 }
 function createMeshForMaterial(doc, mat, i) {
@@ -661,7 +676,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
                     if (shouldOmitLevel) {
                         logger.debug(`[${NAME}] Omitting LOD ${level} for \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                         generatedLODInfos.splice(i, 1);
-                        rmSync(fullPath);
+                        deleteFile(fullPath, opts);
                         i--;
                     }
                     else {
@@ -699,7 +714,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
                 logger.info(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length} (embedded): vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove all LODs. \"${mesh.getName()}\"`);
                 try {
                     for (const lodInfo of generatedLODInfos) {
-                        rmSync(outputDirectory + "/" + lodInfo.path);
+                        deleteFile(outputDirectory + "/" + lodInfo.path, opts);
                     }
                 }
                 catch (err) {
@@ -744,6 +759,9 @@ export function make_progressive_meshes(filePath, _options, opts) {
         }
     });
 }
+function deleteFile(filePath, _context) {
+    rmSync(filePath);
+}
 function clonePrimitiveData(prim, logger) {
     const semantics = prim.listSemantics();
     if (semantics.some(s => s.includes("JOINTS"))) {
@@ -791,6 +809,7 @@ async function createMeshLod(inputFileName, doc, basePath, level, mesh, meshInfo
     // generate hash for mesh
     let hash = mesh["needle:mesh-hash"];
     if (hash === undefined) {
+        logger.info(`[${NAME}] Generating hash for mesh \"${mesh.getName()}\"`);
         hash = getHash(mesh);
         mesh["needle:mesh-hash"] = hash;
     }

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.7.0-alpha";
+export declare const version = "2.7.0-alpha.2";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.7.0-alpha";
+export const version = "2.7.0-alpha.2";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.7.0-alpha",
+  "version": "2.7.0-alpha.2",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",
@@ -24,7 +24,8 @@
   "scripts": {
     "start": "node scripts/index.mjs",
     "dev": "npm-watch compile",
-    "publish": "npm run compile && npm publish --access public",
+    "publish": "npm run compile && npm publish --access public --tag alpha",
+    "publish-tag": "npm dist-tag add @needle-tools/gltf-build-pipeline@2.7.0-alpha.1 alpha",
     "compile": "node tools/prebuild.mjs && (tsc || exit 1) && node tools/postbuild.mjs",
     "clear-caches": "node scripts/clear-caches.mjs",
     "limit-caches": "node scripts/limit-caches.mjs",