@needle-tools/gltf-build-pipeline 1.4.1-beta.4 → 1.4.1-beta.6

package/CHANGELOG.md

@@ -4,10 +4,15 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [1.4.1-beta.4] - 2023-03-15
+## [1.4.1-beta.6] - 2023-03-15
+- add: caching support for mesh LODs
+
+## [1.4.1-beta.5] - 2023-03-15
 - add: auto LODs now calculate mesh density
-- change: generate 6 LOD level with roughly 0.6 ratio of the previous level
+- change: generate 6 LOD level
+- change: use ratio 0.5 of the previous LOD level
 - change: bump gltf-transform to 3.10.1 to fix simplifier
+- change: add sparse transform to progressive transform
 
 ## [1.4.1-beta.3] - 2023-03-14
 - fix: wait for mesh simplifier to be ready

package/dist/cache/cache.d.ts

@@ -1,5 +1,10 @@
 import { ExtensibleProperty } from "@gltf-transform/core";
+/**
+ * Get the available space on the cache directory in bytes.
+ */
 export declare function clearCache(): void;
+export declare function ensureCacheSize(maxSizeMB: number): void;
 export declare function addToCache(key: string, bytes: Uint8Array): void;
 export declare function tryGetFromCache(key: string): Uint8Array | null;
 export declare function getKey(property: ExtensibleProperty): string;
+export declare function getHash(property: ExtensibleProperty | object): number;

package/dist/cache/cache.js

@@ -1,16 +1,59 @@
-import { Texture, Primitive } from "@gltf-transform/core";
+import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
 import tmp from 'tmp';
 import fs from 'fs';
 import path from 'path';
+// import * as CHECKDISCSPACE from 'check-disk-space'
+/**
+ * Get the available space on the cache directory in bytes.
+ */
+// export async function getCacheAvailableSpaceOnDisc(): Promise<number> {
+//     try {
+//         const dir = getCacheDirectory();
+//         const res = await CHECKDISCSPACE.default(dir).catch(() => { return { free: 0 } });
+//         return res.free;
+//     }
+//     catch (e) {
+//         console.error(`Error checking disk space: ${e}`);
+//     }
+//     return 0;
+// }
 export function clearCache() {
     const dir = getCacheDirectory();
     fs.rmSync(dir, { recursive: true });
     console.log(`Cleared cache at ${dir}`);
 }
+export function ensureCacheSize(maxSizeMB) {
+    const dir = getCacheDirectory();
+    let totalSize = 0;
+    const files = fs.readdirSync(dir).map(file => {
+        const path = `${dir}/${file}`;
+        const stats = fs.statSync(path);
+        totalSize += stats.size;
+        return { path, size: stats.size };
+    });
+    const maxSizeInBytes = maxSizeMB * 1024 * 1024;
+    if (totalSize < maxSizeInBytes)
+        return;
+    files.sort((a, b) => {
+        const mtimeA = fs.statSync(a.path).mtimeMs;
+        const mtimeB = fs.statSync(b.path).mtimeMs;
+        return mtimeA - mtimeB;
+    });
+    const targetSize = maxSizeInBytes * 0.9;
+    // remove oldest files first
+    while (totalSize > targetSize) {
+        const file = files.shift();
+        if (!file)
+            break;
+        fs.rmSync(file.path);
+        totalSize -= file.size;
+    }
+    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
+}
 export function addToCache(key, bytes) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
-    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at ${path}`);
+    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
     fs.writeFileSync(path, bytes);
 }
 export function tryGetFromCache(key) {
@@ -23,37 +66,54 @@ export function tryGetFromCache(key) {
     return null;
 }
 export function getKey(property) {
-    let hash = 0;
     let prefix = "prop";
-    const extensions = property.listExtensions();
-    const extras = property.getExtras();
-    // const slots = listTextureSlots(doc, texture);
-    // const channels = getTextureChannelMask(doc, texture);
-    hash += hashObject({
-        extensions,
-        extras,
-    });
     if (property instanceof Texture) {
         prefix = "image";
-        const image = property.getImage();
-        const bytes = image?.buffer;
-        if (bytes)
-            hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
     }
-    else if (property instanceof Primitive) {
-        prefix = "prim";
-        const indices = property.getIndices();
-        const attributes = property.listAttributes();
-        const targets = property.listTargets();
+    let hash = getHash(property);
+    const key = `${prefix}-${hash}`;
+    return key;
+}
+export function getHash(property) {
+    let hash = 0;
+    if (property instanceof ExtensibleProperty) {
+        const extensions = property.listExtensions();
+        const extras = property.getExtras();
+        // const slots = listTextureSlots(doc, texture);
+        // const channels = getTextureChannelMask(doc, texture);
         hash += hashObject({
-            indices,
-            attributes,
-            targets
+            extensions,
+            extras,
         });
+        if (property instanceof Texture) {
+            const image = property.getImage();
+            const bytes = image?.buffer;
+            if (bytes)
+                hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
+        }
+        else if (property instanceof Mesh) {
+            const weights = property.getWeights();
+            hash += hashObject(weights);
+            const primitives = property.listPrimitives();
+            hash += hashObject(primitives);
+        }
+        else if (property instanceof Primitive) {
+            const indices = property.getIndices();
+            const attributes = property.listAttributes();
+            const targets = property.listTargets();
+            const semantics = property.listSemantics();
+            hash += hashObject({
+                indices,
+                attributes,
+                targets,
+                semantics
+            });
+        }
     }
-    let key = hash.toString();
-    key = `${prefix}-${key}`;
-    return key;
+    else {
+        hash += hashObject(property);
+    }
+    return hash;
 }
 function getCacheDirectory() {
     const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache");
@@ -63,6 +123,10 @@ function getCacheDirectory() {
     return tmpDir;
 }
 function hashObject(obj) {
+    if (obj instanceof Accessor) {
+        const buffer = obj.getArray();
+        return hashBuffer(buffer);
+    }
     const str = JSON.stringify(obj);
     let hash = 0;
     for (let i = 0; i < str.length; i++) {

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./utils/index.js";
 export * from "./transforms/index.js";
 export * from "./extensions/index.js";
+export * from "./cache/index.js";

package/dist/index.js

@@ -1,3 +1,4 @@
 export * from "./utils/index.js";
 export * from "./transforms/index.js";
 export * from "./extensions/index.js";
+export * from "./cache/index.js";

package/dist/transforms/needle_mesh_transform.js

@@ -5,6 +5,7 @@ import { NEEDLE_mesh_compression } from "../extensions/index.js";
 const NAME = 'needle_mesh_transform';
 export const needle_mesh_transform = function (_options) {
     return async (document, _context) => {
+        // TODO: needs to be moved into NEEDLE_progressive
         await simplifierTransform(document, _context, _options);
         await applyMeshCompression(document, _context);
     };

package/dist/transforms/needle_progressive.d.ts

@@ -32,4 +32,7 @@ export declare enum TextureResizeFilter {
 export declare const TEXTURE_RESIZE_DEFAULTS: TextureResizeOptions;
 export declare function resizeTextureToMax(texture: Texture, maxSize: number, logger: ILogger): Promise<void>;
 export declare function make_progressive_meshes(filePath: string, _options: any, opts?: ProgressiveOptions): Transform;
+export declare function isMeshLOD(filepath: string): boolean;
+export declare function isImageLOD(filepath: string): boolean;
+export declare function isLOD(filepath: string): boolean;
 export {};

package/dist/transforms/needle_progressive.js

@@ -1,10 +1,10 @@
 import { BufferUtils, Document, NodeIO, PropertyType } from '@gltf-transform/core';
-import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots } from '@gltf-transform/functions';
+import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots, sparse } from '@gltf-transform/functions';
 import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive, generateGuid } from "../extensions/NEEDLE_progressive/index.js";
 import { createOpaqueExtension, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
 import path from 'path';
-import { existsSync, rmSync, statSync } from 'fs';
-import { addToCache, getKey, tryGetFromCache } from '../cache/index.js';
+import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
+import { addToCache, getHash, getKey, tryGetFromCache } from '../cache/index.js';
 import sharp from "sharp";
 import { foreachGLTF } from '../utils/index.js';
 import { needle_asset } from './needle_asset.js';
@@ -67,7 +67,7 @@ async function process(file, opts) {
             }
             //@ts-ignore
             document.getLogger().verbosity = 0;
-            await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts), make_progressive_meshes(file, {}, opts));
+            await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts), make_progressive_meshes(file, {}, opts), sparse());
             let filename = file;
             if (opts.debug === true) {
                 // get extension
@@ -314,8 +314,8 @@ export function make_progressive_meshes(filePath, _options, opts) {
         for (let l = 0; l < MAX_LODS; l++) {
             const t = l / MAX_LODS;
             const latest = LODS[LODS.length - 1];
-            const ratio = latest.ratio * .6;
-            const error = .1 + t * 0.4;
+            const ratio = latest.ratio * .5;
+            const error = .05 + t * .5;
             LODS.push({
                 ratio: ratio,
                 error,
@@ -357,10 +357,9 @@ export function make_progressive_meshes(filePath, _options, opts) {
                     const lod = LODS[level];
                     const guid = settings.guid;
                     // TODO: check if the LOD level did actually change the mesh - otherwise we can just skip generating lower resolutions
-                    const lodTask = createMeshLod(doc, outputDirectory, level, mesh, guid, lod);
+                    const lodTask = createMeshLod(doc, outputDirectory, level, mesh, guid, lod, opts);
                     lodTask.then(res => {
-                        console.log(`${NAME}: ${mesh.getName()} Register LOD ${level}`);
-                        if (res.success) {
+                        if (res?.success) {
                             generatedLODs[level] = res.path;
                         }
                     });
@@ -428,7 +427,29 @@ export function make_progressive_meshes(filePath, _options, opts) {
  * @param guid a unique identifier for the mesh
  * @param error the error to use for simplification. Higher means more simplification
  */
-async function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
+async function createMeshLod(doc, basePath, level, mesh, guid, lodLevel, opts) {
+    const fileExtension = ".glb";
+    const fileName_withoutExtension = "mesh_lod_" + level + "_" + guid;
+    const fileName = fileName_withoutExtension + fileExtension;
+    const outPath = basePath + fileName;
+    const primitives = mesh.listPrimitives();
+    let cacheKey = undefined;
+    if (opts.useCache !== false) {
+        // generate hash for mesh
+        let hash = mesh["needle:mesh-hash"];
+        if (hash === undefined) {
+            hash = getHash(mesh);
+            mesh["needle:mesh-hash"] = hash;
+        }
+        hash += getHash(lodLevel);
+        cacheKey = `mesh_lod_${level}_${hash}`;
+        const cachedValue = tryGetFromCache(cacheKey);
+        if (cachedValue) {
+            console.log(`${NAME}: ${mesh.getName()} LOD ${level} loaded ${(cachedValue.length / 1024).toFixed()} KB from cache`);
+            writeFileSync(outPath, cachedValue);
+            return { path: fileName, success: true };
+        }
+    }
     const newDoc = new Document();
     const newScene = newDoc.createScene();
     newDoc.createBuffer();
@@ -439,11 +460,10 @@ async function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
     const newMesh = newDoc.createMesh();
     newMesh.setName(mesh.getName() + " LOD " + level);
     newNode.setMesh(newMesh);
-    const primitives = mesh.listPrimitives();
+    const newMat = newDoc.createMaterial();
     const tasks = new Array();
     // let didRunSimplification = false;
     for (const prim of primitives) {
-        const newMat = newDoc.createMaterial();
         const newPrim = newDoc.createPrimitive();
         newMesh.addPrimitive(newPrim);
         newPrim.setName(prim.getName());
@@ -475,6 +495,8 @@ async function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
             console.log(`${NAME}: Copy primitive ${name} extension`);
             newPrim.setExtension(name, copy);
         }
+    }
+    for (const newPrim of newMesh.listPrimitives()) {
         if (lodLevel.error != undefined || lodLevel.ratio != undefined) {
             if (lodLevel.error > 0 || lodLevel.ratio < 1) {
                 // didRunSimplification = true;
@@ -484,6 +506,7 @@ async function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
         }
     }
     await Promise.all(tasks);
+    tasks.length = 0;
     // if we run simplify but the mesh didnt change
     // if (didRunSimplification) {
     // 	let anyPrimitiveDidChange = false;
@@ -511,13 +534,14 @@ async function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
     }
     const newIo = new NodeIO();
     registerExtensions(newIo);
-    const ext = ".glb";
-    const name = "mesh_lod_" + level + "_" + guid;
-    const filename = name + ext;
-    const outName = basePath + filename;
-    console.log(`${NAME}: Write mesh LOD ${level} to \"${filename}\" with: ${JSON.stringify(lodLevel)}`);
-    await newIo.write(outName, newDoc);
-    return { path: filename, success: true };
+    console.log(`${NAME}: Write mesh LOD ${level} to \"${fileName}\" with: ${JSON.stringify(lodLevel)}`);
+    await newIo.write(outPath, newDoc);
+    // add to cache
+    if (cacheKey != undefined) {
+        const bytes = readFileSync(outPath);
+        addToCache(cacheKey, bytes);
+    }
+    return { path: fileName, success: true };
 }
 function calculateMeshDensity(prim) {
     const bounds = {
@@ -612,9 +636,12 @@ async function simplifyMesh(doc, mesh, prim, lodinfo) {
     // 	simplifyOptions.lockBorder = lodinfo.lockBorder;
     return compressPrimitive(doc, mesh, prim, weldOptions, simplifyOptions);
 }
-function isMeshLOD(filepath) {
+export function isMeshLOD(filepath) {
     return filepath.includes("mesh_lod_");
 }
-function isImageLOD(filepath) {
+export function isImageLOD(filepath) {
     return filepath.includes("image_");
 }
+export function isLOD(filepath) {
+    return isMeshLOD(filepath) || isImageLOD(filepath);
+}

package/index.mjs

@@ -1,6 +1,7 @@
 
-
 import { spawnSync } from "child_process";
+import { getVersion } from "./dist/index.js";
+
 const workingDir = process.cwd();
 
 // Original invoked dir set by npm
@@ -28,6 +29,13 @@ function runWithArgs(cmd) {
     spawnSync(cmd + " " + commandline_args, defaultArgs);
 }
 
-
-runWithArgs("node make-progressive.mjs");
-runWithArgs("node pack-gltf.mjs");
+try {
+    const startTime = Date.now();
+    runWithArgs("node make-progressive.mjs");
+    runWithArgs("node pack-gltf.mjs");
+    const durationInMinutes = ((Date.now() - startTime) / 1000 / 60).toFixed(1);
+    console.log(`[Needle Build Pipeline] v${getVersion()} — Finished progressive transform and optimization in ${durationInMinutes} minutes.`)
+}
+catch (err) {
+    console.error("Failed running commandline args:", err.message);
+}

package/pack-gltf.mjs

@@ -1,20 +1,18 @@
-import glob from 'glob';
 import path from 'path';
 import { NodeIO } from '@gltf-transform/core';
 import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
-import { draco } from '@gltf-transform/functions';
 import draco3d from 'draco3dgltf';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS } from "./dist/extensions/NEEDLE_opaque/index.js";
 import commandLineArgs from "command-line-args";
 import { NEEDLE_mesh_compression, NEEDLE_compression_texture } from './dist/extensions/index.js';
-import { existsSync, statSync } from 'fs';
+import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { needle_mesh_transform } from "./dist/transforms/needle_mesh_transform.js";
 import { needle_texture_transform } from "./dist/transforms/needle_texture_transform.js";
 import { needle_animation_transform } from "./dist/transforms/needle_animation_transform.js";
 import { foreachGLTF } from './fileutils.mjs';
 import { needle_asset } from './dist/transforms/needle_asset.js';
 import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
-import { getVersion } from './dist/index.js';
+import { getVersion, getHash, addToCache, tryGetFromCache, isLOD, ensureCacheSize as ensureMaxCacheSize, isMeshLOD } from './dist/index.js';
 
 console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`)
 
@@ -77,6 +75,9 @@ let totalFilesProcessed = 0;
 let totalFileSizeInMB = 0;
 let totalFileSizeInMBBefore = 0;
 
+/** Compress a glTF file
+ * @param {string} file
+ */
 async function packFile(file) {
     if (!existsSync(file)) {
         console.error("WARN: can not pack, file does not exist: " + file);
@@ -86,6 +87,22 @@ async function packFile(file) {
         return;
     }
 
+    // check if it's in the cache. 
+    // We only need to check mesh LOD files because image LODs are cache via our toktx transform
+    // We can not cache the main glTF files because the hash always changes for them
+    let cacheKey = undefined;
+    if (allowCaching && isMeshLOD(file)) {
+        const existing = readFileSync(file);
+        const hash = getHash(existing);
+        cacheKey = "packed-" + hash;
+        const cached = tryGetFromCache(cacheKey);
+        if (cached) {
+            console.log("→ Skip packing (loaded from cache): " + path.basename(file));
+            writeFileSync(file, cached);
+            return;
+        }
+    }
+
     if (packedFiles.has(file)) return;
     packedFiles.add(file);
     console.log("→ Begin packing: " + file);
@@ -123,6 +140,10 @@ async function packFile(file) {
         );
         if (debug) file += ".debug.glb";
         await io.write(file, document);
+
+        if (allowCaching && cacheKey != undefined && existsSync(file))
+            addToCache(cacheKey, readFileSync(file));
+
         const duration = Date.now() - startTime;
         const sizeInMB = statSync(file).size / 1024 / 1024;
         const filename = path.basename(file);
@@ -148,8 +169,17 @@ let startTime = Date.now();
 const inputArgs = args.slice(2);
 foreachGLTF(inputArgs, async (file) => {
     await packFile(file);
-}).then(() => {
+}).then(async () => {
     const duration = Date.now() - startTime;
     const durationInMinutes = duration / 1000 / 60;
+    if (allowCaching) {
+        // if the user's cache directory has more than 25GB free space, we allow a cache of 2 GB - otherwise 1 GB
+        // const freeSpaceInMB = (await getCacheAvailableSpaceOnDisc().catch(() => 0)) / 1024 / 1024;
+        // const maxCacheSize = freeSpaceInMB > 25000 ? 2000 : 1000;
+        ensureMaxCacheSize(1000);
+    }
     console.log(`[Needle Build Pipeline] v${getVersion()} — Finished packing ${totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${totalFileSizeInMBBefore.toFixed(1)} MB → ${totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${totalGPUMemoryInMB.toFixed(1)} MB`);
-})
+}).catch(err => {
+    console.error("ERR: packing failed: " + err.message);
+    if (debug) throw err;
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.4.1-beta.4",
+  "version": "1.4.1-beta.6",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",