@needle-tools/gltf-build-pipeline 1.4.1-beta.1 → 1.4.1-beta.11

package/CHANGELOG.md

@@ -4,10 +4,40 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [1.4.1-beta] - 2023-03-13
+## [1.4.1-beta.11] - 2023-03-20
+- add: NEEDLE_progressive LOD meshes morph targets data
+- fix: NEEDLE_progressive should check if simplifcation did result in improved meshes (> 32 verts removed && > 5 % improvement) and otherwise discard LODs (and the whole extension if no improvement was made)
+- change: Unity NEEDLE_progressive logs
+
+## [1.4.1-beta.10] - 2023-03-19
+- change: NEEDLE_progressive calculates density per mesh level
+- change: NEEDLE_progressive `uri` field is removed and added `lod` array of objects
+
+## [1.4.1-beta.8] - 2023-03-18
+- change: if an asset doesnt have a guid we use a uuid package now
+- change: ignore mesh LODs with vertex count < 4
+- change: allow generating mesh LODs without extension (if the mesh has > 32 vertices)
+- change: omit generated LODs if the change is too small (e.g. the vertex count didnt change enough)
+
+## [1.4.1-beta.6] - 2023-03-15
+- add: caching support for mesh LODs
+
+## [1.4.1-beta.5] - 2023-03-15
+- add: auto LODs now calculate mesh density
+- change: generate 6 LOD level
+- change: use ratio 0.5 of the previous LOD level
+- change: bump gltf-transform to 3.10.1 to fix simplifier
+- change: add sparse transform to progressive transform
+
+## [1.4.1-beta.3] - 2023-03-14
+- fix: wait for mesh simplifier to be ready
+
+## [1.4.1-beta.2] - 2023-03-13
 - change: update gltf-transform to `^3.10.0`
 - change: progressive mesh transform does now set ratio for LODs (see [simplifier issue 661 on github](https://github.com/zeux/meshoptimizer/issues/661))
 - fix: issue where progressive meshes were not generated if a mesh was multiple times in the glTF
+- change: simplify to use ratio, fix issue where not all meshes were processed
+- change: auto LOD max error set to 0.1
 
 ## [1.4.0-beta] - 2023-03-12
 - add: progressive mesh transform to automatically generate mesh LOD levels (currently we only generate one low poly version)

package/dist/cache/cache.d.ts

@@ -1,5 +1,10 @@
 import { ExtensibleProperty } from "@gltf-transform/core";
+/**
+ * Get the available space on the cache directory in bytes.
+ */
 export declare function clearCache(): void;
+export declare function ensureCacheSize(maxSizeMB: number): void;
 export declare function addToCache(key: string, bytes: Uint8Array): void;
 export declare function tryGetFromCache(key: string): Uint8Array | null;
 export declare function getKey(property: ExtensibleProperty): string;
+export declare function getHash(property: ExtensibleProperty | object): number;

package/dist/cache/cache.js

@@ -1,16 +1,59 @@
-import { Texture, Primitive } from "@gltf-transform/core";
+import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
 import tmp from 'tmp';
 import fs from 'fs';
 import path from 'path';
+// import * as CHECKDISCSPACE from 'check-disk-space'
+/**
+ * Get the available space on the cache directory in bytes.
+ */
+// export async function getCacheAvailableSpaceOnDisc(): Promise<number> {
+//     try {
+//         const dir = getCacheDirectory();
+//         const res = await CHECKDISCSPACE.default(dir).catch(() => { return { free: 0 } });
+//         return res.free;
+//     }
+//     catch (e) {
+//         console.error(`Error checking disk space: ${e}`);
+//     }
+//     return 0;
+// }
 export function clearCache() {
     const dir = getCacheDirectory();
     fs.rmSync(dir, { recursive: true });
     console.log(`Cleared cache at ${dir}`);
 }
+export function ensureCacheSize(maxSizeMB) {
+    const dir = getCacheDirectory();
+    let totalSize = 0;
+    const files = fs.readdirSync(dir).map(file => {
+        const path = `${dir}/${file}`;
+        const stats = fs.statSync(path);
+        totalSize += stats.size;
+        return { path, size: stats.size };
+    });
+    const maxSizeInBytes = maxSizeMB * 1024 * 1024;
+    if (totalSize < maxSizeInBytes)
+        return;
+    files.sort((a, b) => {
+        const mtimeA = fs.statSync(a.path).mtimeMs;
+        const mtimeB = fs.statSync(b.path).mtimeMs;
+        return mtimeA - mtimeB;
+    });
+    const targetSize = maxSizeInBytes * 0.9;
+    // remove oldest files first
+    while (totalSize > targetSize) {
+        const file = files.shift();
+        if (!file)
+            break;
+        fs.rmSync(file.path);
+        totalSize -= file.size;
+    }
+    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
+}
 export function addToCache(key, bytes) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
-    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at ${path}`);
+    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
     fs.writeFileSync(path, bytes);
 }
 export function tryGetFromCache(key) {
@@ -23,37 +66,54 @@ export function tryGetFromCache(key) {
     return null;
 }
 export function getKey(property) {
-    let hash = 0;
     let prefix = "prop";
-    const extensions = property.listExtensions();
-    const extras = property.getExtras();
-    // const slots = listTextureSlots(doc, texture);
-    // const channels = getTextureChannelMask(doc, texture);
-    hash += hashObject({
-        extensions,
-        extras,
-    });
     if (property instanceof Texture) {
         prefix = "image";
-        const image = property.getImage();
-        const bytes = image?.buffer;
-        if (bytes)
-            hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
     }
-    else if (property instanceof Primitive) {
-        prefix = "prim";
-        const indices = property.getIndices();
-        const attributes = property.listAttributes();
-        const targets = property.listTargets();
+    let hash = getHash(property);
+    const key = `${prefix}-${hash}`;
+    return key;
+}
+export function getHash(property) {
+    let hash = 0;
+    if (property instanceof ExtensibleProperty) {
+        const extensions = property.listExtensions();
+        const extras = property.getExtras();
+        // const slots = listTextureSlots(doc, texture);
+        // const channels = getTextureChannelMask(doc, texture);
         hash += hashObject({
-            indices,
-            attributes,
-            targets
+            extensions,
+            extras,
         });
+        if (property instanceof Texture) {
+            const image = property.getImage();
+            const bytes = image?.buffer;
+            if (bytes)
+                hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
+        }
+        else if (property instanceof Mesh) {
+            const weights = property.getWeights();
+            hash += hashObject(weights);
+            const primitives = property.listPrimitives();
+            hash += hashObject(primitives);
+        }
+        else if (property instanceof Primitive) {
+            const indices = property.getIndices();
+            const attributes = property.listAttributes();
+            const targets = property.listTargets();
+            const semantics = property.listSemantics();
+            hash += hashObject({
+                indices,
+                attributes,
+                targets,
+                semantics
+            });
+        }
     }
-    let key = hash.toString();
-    key = `${prefix}-${key}`;
-    return key;
+    else {
+        hash += hashObject(property);
+    }
+    return hash;
 }
 function getCacheDirectory() {
     const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache");
@@ -63,6 +123,10 @@ function getCacheDirectory() {
     return tmpDir;
 }
 function hashObject(obj) {
+    if (obj instanceof Accessor) {
+        const buffer = obj.getArray();
+        return hashBuffer(buffer);
+    }
     const str = JSON.stringify(obj);
     let hash = 0;
     for (let i = 0; i < str.length; i++) {

package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js

@@ -44,6 +44,8 @@ export class NEEDLE_mesh_compression extends Extension {
         return this;
     }
     write(ctx) {
+        if (debugLog)
+            console.log("WRITE " + EXTENSION_NAME);
         const root = this.document.getRoot();
         const rootExtension = root.getExtension(EXTENSION_NAME);
         if (rootExtension) {
@@ -72,7 +74,7 @@ export class NEEDLE_mesh_compression extends Extension {
 export class NEEDLE_mesh_compression_property extends ExtensionProperty {
     static EXTENSION_NAME = EXTENSION_NAME;
     extensionName = EXTENSION_NAME;
-    parentTypes = [PropertyType.PRIMITIVE, PropertyType.ROOT];
+    parentTypes = [PropertyType.ROOT, PropertyType.PRIMITIVE, PropertyType.MESH];
     propertyType = EXTENSION_NAME;
     init() { }
     model;

package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.d.ts

@@ -1,15 +1,21 @@
 import { Extension, ExtensionProperty, Graph, IProperty, Property, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare const NEEDLE_PROGRESSIVE_EXTENSION_NAME = "NEEDLE_progressive";
-export declare interface IProgressiveTextureInfo {
+export declare interface IProgressiveLoadingInfo {
+    /** guid of the asset the LODs belong to */
     guid: string;
-    uri?: string | Array<string>;
+    /** paths to the LOD level files */
+    lods?: Array<{
+        path: string;
+        /** the density of the lod level */
+        density?: number;
+    }>;
+    density?: number;
 }
-export declare function generateGuid(): string;
 export declare class NEEDLE_progressive extends Extension {
     static EXTENSION_NAME: string;
     extensionName: string;
-    static create(graph: Graph<Property<IProperty>>, data: IProgressiveTextureInfo): NEEDLE_progressive_texture_property;
-    create(data: IProgressiveTextureInfo): NEEDLE_progressive_texture_property;
+    static create(graph: Graph<Property<IProperty>>, data: IProgressiveLoadingInfo): NEEDLE_progressive_texture_property;
+    create(data: IProgressiveLoadingInfo): NEEDLE_progressive_texture_property;
     read(_context: ReaderContext): this;
     write(context: WriterContext): this;
 }
@@ -19,5 +25,5 @@ export declare class NEEDLE_progressive_texture_property extends ExtensionProper
     parentTypes: PropertyType[];
     propertyType: string;
     init(): void;
-    model: IProgressiveTextureInfo;
+    model: IProgressiveLoadingInfo;
 }

package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.js

@@ -1,10 +1,6 @@
-import { Extension, ExtensionProperty, PropertyType, uuid } from "@gltf-transform/core";
+import { Extension, ExtensionProperty, PropertyType } from "@gltf-transform/core";
 // TODO: rename to NEEDLE_progressive_texture
 export const NEEDLE_PROGRESSIVE_EXTENSION_NAME = "NEEDLE_progressive";
-export function generateGuid() {
-    // TODO: replace with same uuid package as in needle engine
-    return uuid();
-}
 export class NEEDLE_progressive extends Extension {
     static EXTENSION_NAME = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     extensionName = NEEDLE_PROGRESSIVE_EXTENSION_NAME;

package/dist/extensions/NEEDLE_progressive_mesh_settings/index.d.ts

@@ -1,6 +1,7 @@
 import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare interface IProgressiveMeshSettings {
     guid: string;
+    generateLods: boolean;
 }
 export declare class NEEDLE_progressive_mesh_settings extends Extension {
     static EXTENSION_NAME: string;
@@ -14,7 +15,6 @@ export declare class NEEDLE_progressive_mesh_settings_property extends Extension
     parentTypes: PropertyType[];
     propertyType: string;
     init(): void;
-    get isValid(): boolean;
-    maxSize: number;
+    generateLods: boolean;
     guid: string;
 }

package/dist/extensions/NEEDLE_progressive_mesh_settings/index.js

@@ -36,9 +36,6 @@ export class NEEDLE_progressive_mesh_settings_property extends ExtensionProperty
     parentTypes = [PropertyType.MESH];
     propertyType = EXTENSION_NAME;
     init() { }
-    get isValid() {
-        return this.maxSize !== undefined && this.maxSize > 0;
-    }
-    maxSize;
+    generateLods;
     guid;
 }

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./utils/index.js";
 export * from "./transforms/index.js";
 export * from "./extensions/index.js";
+export * from "./cache/index.js";

package/dist/index.js

@@ -1,3 +1,4 @@
 export * from "./utils/index.js";
 export * from "./transforms/index.js";
 export * from "./extensions/index.js";
+export * from "./cache/index.js";

package/dist/transforms/needle_mesh_transform.d.ts

@@ -1,4 +1,4 @@
 import { Transform, Document, Primitive, Mesh } from "@gltf-transform/core";
 import { SimplifyOptions, WeldOptions } from "@gltf-transform/functions";
 export declare const needle_mesh_transform: (_options: SimplifyOptions) => Transform;
-export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): void;
+export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, index: number, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<Primitive>;

package/dist/transforms/needle_mesh_transform.js

@@ -5,6 +5,7 @@ import { NEEDLE_mesh_compression } from "../extensions/index.js";
 const NAME = 'needle_mesh_transform';
 export const needle_mesh_transform = function (_options) {
     return async (document, _context) => {
+        // TODO: needs to be moved into NEEDLE_progressive
         await simplifierTransform(document, _context, _options);
         await applyMeshCompression(document, _context);
     };
@@ -49,7 +50,9 @@ async function simplifierTransform(document, _context, _options) {
         overwrite: true,
     };
     for (const mesh of document.getRoot().listMeshes()) {
+        let i = 0;
         for (const prim of mesh.listPrimitives()) {
+            const index = i++;
             // TODO: mesh compression caching
             // const key = getKey(prim);
             // const cached = tryGetFromCache(key);
@@ -83,17 +86,31 @@ async function simplifierTransform(document, _context, _options) {
                 logger.warn(`[${NAME}]: Skipping primitive of mesh "${mesh.getName()}": Requires TRIANGLES draw mode.`);
                 continue;
             }
-            compressPrimitive(document, mesh, prim, weldOptions, simplifyOpts);
+            compressPrimitive(document, mesh, prim, index, weldOptions, simplifyOpts);
         }
     }
     return true;
 }
-export function compressPrimitive(document, mesh, prim, weldOptions, meshOptions) {
+export async function compressPrimitive(document, mesh, prim, index, weldOptions, meshOptions) {
     const logger = document.getLogger();
-    logger.debug(`[${NAME}]: Welding primitive of mesh "${mesh.getName()}" → tolerance: ${weldOptions.tolerance}`);
+    logger.debug(`weld: primitive[${index}] of mesh "${mesh.getName()}" → tolerance: ${weldOptions.tolerance}`);
     weldPrimitive(document, prim, weldOptions);
-    logger.debug(`[${NAME}]: Simplifying primitive of mesh "${mesh.getName()}" → ratio: ${meshOptions.ratio}, error: ${meshOptions.error}, lockBorder: ${meshOptions.lockBorder}`);
-    simplifyPrimitive(document, prim, meshOptions);
+    // make sure we wait for the mesh simplifier to be ready
+    if (meshOptions.simplifier && meshOptions.simplifier?.ready instanceof Promise) {
+        const res = await meshOptions.simplifier?.ready
+            .then(() => {
+            // ready!
+            return true;
+        })
+            .catch((err) => {
+            console.error(`[${NAME}]: Simplifier failed to load...\n${err}`);
+            return false;
+        });
+        if (!res)
+            return;
+    }
+    logger.debug(`simplify: primitive[${index}] of mesh "${mesh.getName()}" → ratio: ${meshOptions.ratio}, error: ${meshOptions.error}, lockBorder: ${meshOptions.lockBorder}`);
+    return simplifyPrimitive(document, prim, meshOptions);
 }
 // function updatePrimitive(prim: Primitive, simplifier: any, positionArray, dstIndicesArray) {
 // 	const position = prim.getAttribute('POSITION')!;

package/dist/transforms/needle_progressive.d.ts

@@ -32,4 +32,7 @@ export declare enum TextureResizeFilter {
 export declare const TEXTURE_RESIZE_DEFAULTS: TextureResizeOptions;
 export declare function resizeTextureToMax(texture: Texture, maxSize: number, logger: ILogger): Promise<void>;
 export declare function make_progressive_meshes(filePath: string, _options: any, opts?: ProgressiveOptions): Transform;
+export declare function isMeshLOD(filepath: string): boolean;
+export declare function isImageLOD(filepath: string): boolean;
+export declare function isLOD(filepath: string): boolean;
 export {};

package/dist/transforms/needle_progressive.js

@@ -1,15 +1,16 @@
 import { BufferUtils, Document, NodeIO, PropertyType } from '@gltf-transform/core';
-import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots } from '@gltf-transform/functions';
-import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive, generateGuid } from "../extensions/NEEDLE_progressive/index.js";
+import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots, sparse } from '@gltf-transform/functions';
+import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive } from "../extensions/NEEDLE_progressive/index.js";
 import { createOpaqueExtension, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
 import path from 'path';
-import { existsSync } from 'fs';
-import { addToCache, getKey, tryGetFromCache } from '../cache/index.js';
+import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
+import { addToCache, getHash, getKey, tryGetFromCache } from '../cache/index.js';
 import sharp from "sharp";
-import { foreachGLTF } from '../utils/index.js';
+import { foreachGLTF, generateGuid } from '../utils/index.js';
 import { needle_asset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive } from './needle_mesh_transform.js';
+import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
 export async function make_progressive(opts) {
     console.log("INFO: Run progressive transform: " + opts.path);
     const file = opts.path;
@@ -67,7 +68,7 @@ async function process(file, opts) {
             }
             //@ts-ignore
             document.getLogger().verbosity = 0;
-            await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts), make_progressive_meshes(file, {}, opts));
+            await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts), make_progressive_meshes(file, {}, opts), sparse());
             let filename = file;
             if (opts.debug === true) {
                 // get extension
@@ -109,40 +110,40 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             const uri = texture.getURI();
             const match = !options.pattern || options.pattern.test(name) || options.pattern.test(uri);
             if (!match) {
-                logger.debug(`${NAME}: Skipping, excluded by "pattern" parameter. ${textureIndex}`);
+                logger.debug(`[${NAME}] Skipping, excluded by "pattern" parameter. ${textureIndex}`);
                 continue;
             }
             if (texture.getMimeType() !== 'image/png' && texture.getMimeType() !== 'image/jpeg') {
-                logger.warn(`${NAME}: Skipping, unsupported texture type "${texture.getMimeType()} at ${textureIndex}".`);
+                logger.warn(`[${NAME}] Skipping, unsupported texture type "${texture.getMimeType()} at ${textureIndex}".`);
                 continue;
             }
             const slots = listTextureSlots(texture);
             if (options.slots && !slots.some((slot) => options.slots?.test(slot))) {
-                logger.debug(`${NAME}: Skipping, [${slots.join(', ')}] excluded by "slots" parameter. ${textureIndex}`);
+                logger.debug(`[${NAME}] Skipping, [${slots.join(', ')}] excluded by "slots" parameter. ${textureIndex}`);
                 continue;
             }
             const settings = texture.getExtension(NEEDLE_progressive_texture_settings.EXTENSION_NAME);
             if (!settings) {
-                logger.warn(`${NAME}: Skipping, no settings found ${textureIndex}`);
+                logger.warn(`[${NAME}] Skipping, no settings found ${textureIndex}`);
                 continue;
             }
             if (!settings.isValid) {
-                logger.warn(`${NAME}: Skipping, settings are invalid ${textureIndex}`);
+                logger.warn(`[${NAME}] Skipping, settings are invalid ${textureIndex}`);
                 continue;
             }
             options.size[0] = settings.maxSize;
             options.size[1] = settings.maxSize;
             const existingExtension = texture.getExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME);
             if (existingExtension !== undefined && existingExtension !== null) {
-                logger.debug(`${NAME}: Skipping, texture is already progressive ${textureIndex}`);
+                logger.debug(`[${NAME}] Skipping, texture is already progressive ${textureIndex}`);
                 continue;
             }
             const progressiveExt = doc.createExtension(NEEDLE_progressive);
             if (!progressiveExt || !progressiveExt.create) {
-                logger.debug(`${NAME}: Skipping because progressive extension is not available. ${textureIndex}`);
+                logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${textureIndex}`);
                 continue;
             }
-            const textureGuid = settings.guid?.length ? settings.guid : generateGuid();
+            const textureGuid = settings.guid?.length ? settings.guid : generateGuid(path + "_texture_" + index.toString());
             const outPath = createTextureLod(textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
             if (!outPath) {
                 continue;
@@ -151,7 +152,11 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             // TODO: this path should be added to the NEEDLE dependencies extension
             texture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
                 guid: textureGuid,
-                uri: relName,
+                lods: [
+                    {
+                        path: relName,
+                    }
+                ],
             }));
             const key = getKey(texture);
             if (opts.useCache) {
@@ -262,7 +267,7 @@ async function resizeTexture(options, texture, logger, uri, slots) {
     const [maxWidth, maxHeight] = options.size;
     const [srcWidth, srcHeight] = texture.getSize();
     if (srcWidth <= maxWidth && srcHeight <= maxHeight) {
-        logger.debug(`${NAME}: Skipping, the texture is already smaller than the requested size ${maxWidth}x${maxHeight}}`);
+        logger.debug(`[${NAME}] Skipping, the texture is already smaller than the requested size ${maxWidth}x${maxHeight}}`);
         return;
     }
     if (maxWidth <= 0 || maxHeight <= 0) {
@@ -298,70 +303,203 @@ export function make_progressive_meshes(filePath, _options, opts) {
             console.log("Progressive Mesh Transform: skip lod at " + filePath);
             return Promise.resolve();
         }
+        await MeshoptSimplifier.ready;
         const logger = doc.getLogger();
         const filename = path.basename(filePath);
-        console.log("> Progressive Mesh Transform: " + filename);
+        console.log(`[${NAME}]: Begin generating progressive meshes for \"${filename}\"`);
         let index = 0;
         // the lowest LOD (last entry) will be saved in the original file
+        const MAX_LODS = 6;
         const LODS = [
-            { error: 0 },
-            { ratio: 0.5, error: 1 },
-            { ratio: 0.25, error: 1 },
+            {
+                ratio: 1,
+                error: 0,
+                lockBorder: true
+            },
         ];
+        for (let l = 0; l < MAX_LODS; l++) {
+            const t = l / MAX_LODS;
+            const latest = LODS[LODS.length - 1];
+            const ratio = latest.ratio * .5;
+            const error = .05 + t * .5;
+            LODS.push({
+                ratio: ratio,
+                error,
+                lockBorder: true,
+                // toleranceNormal: 0.1 + t * 0.1,
+            });
+        }
         /** Map of visited GUID -> extension */
         const visited = new Map();
         const meshes = doc.getRoot().listMeshes();
         for (const mesh of meshes) {
-            // TODO: skip meshes that are already below a certain threshold (E.g. just 4 vertices)
-            const settings = mesh.getExtension(NEEDLE_progressive_mesh_settings.EXTENSION_NAME);
+            /** The size and vertex count of the original mesh */
+            const originalMeshInformation = getMeshInformation(mesh);
+            const { totalVertexCount: originalTotalVertexCount, maxVertexCount: originalMaxVertexCount } = originalMeshInformation;
+            let settings = mesh.getExtension(NEEDLE_progressive_mesh_settings.EXTENSION_NAME);
             if (!settings) {
-                logger.debug(`${NAME}: no ${NEEDLE_progressive_mesh_settings.EXTENSION_NAME} found for mesh at index ${index++}`);
+                if (originalMaxVertexCount < 32) {
+                    logger.debug(`[${NAME}] Skipping, no settings found and no primitive with more than 32 vertices. mesh[${index}] has ${originalTotalVertexCount} vertices (max: ${originalMaxVertexCount})`);
+                    continue;
+                }
+                else {
+                    settings = {
+                        guid: generateGuid(path + "_mesh_" + index.toString()),
+                        generateLods: true,
+                    };
+                    logger.debug(`[${NAME}] No settings found for mesh[${index}] with ${originalTotalVertexCount} vertices (max: ${originalMaxVertexCount}) - will use default settings (guid: ${settings.guid})`);
+                }
+            }
+            else if (settings.generateLods === false) {
+                logger.debug(`[${NAME}] progressive mesh disabled for mesh at index ${index++}`);
                 continue;
             }
             const progressiveExt = doc.createExtension(NEEDLE_progressive);
             if (!progressiveExt || !progressiveExt.create) {
-                logger.debug(`${NAME}: Skipping because progressive extension is not available. ${index++}`);
+                logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${index++}`);
                 continue;
             }
             // check if we have already produced LODs for this mesh
             let needsToGenerateLODs = true;
-            const generatedLODs = new Array();
+            const generatedLODInfos = new Array();
             if (visited.has(settings.guid)) {
                 // we can not just set the extension here...
-                logger.debug(`${NAME}: Mesh has already been processed ${index} - will use previously generated LODs`);
+                logger.debug(`[${NAME}] Mesh has already been processed ${index} - will use previously generated LODs`);
                 needsToGenerateLODs = false;
-                const existingPaths = visited.get(settings.guid).paths;
-                generatedLODs.push(...existingPaths);
+                const existingPaths = visited.get(settings.guid);
+                generatedLODInfos.push(...existingPaths.infos);
             }
+            /** async simplification tasks... */
+            const tasks = new Array();
             if (needsToGenerateLODs) {
+                console.log(`[${NAME}] Begin generating mesh LODs for \"${mesh.getName()}\"`);
                 // produce LODS, skip the last level because it will be saved in the original file
                 for (let level = 0; level < LODS.length - 1; level++) {
                     const lod = LODS[level];
-                    console.log(`${NAME}: Generate LOD ${level}`);
                     const guid = settings.guid;
-                    const lodPath = createMeshLod(doc, outputDirectory, level, mesh, guid, lod);
-                    generatedLODs.push(lodPath);
+                    // TODO: check if the LOD level did actually change the mesh - otherwise we can just skip generating lower resolutions
+                    const lodTask = createMeshLod(doc, outputDirectory, level, mesh, originalMeshInformation, guid, lod, opts);
+                    lodTask.then(res => {
+                        if (res?.success) {
+                            generatedLODInfos[level] = res;
+                        }
+                    });
+                    tasks.push(lodTask);
+                }
+                // wait for all LODs to be generated
+                await Promise.all(tasks);
+                tasks.length = 0;
+            }
+            // compare LOD results and ignore if the mesh did not change
+            // TODO: this should actually check mesh semantics (perhaps hash the mesh using the caching system?)
+            let prevSize = -Infinity;
+            for (let i = 0; i < generatedLODInfos.length; i++) {
+                const lodInfo = generatedLODInfos[i];
+                if (!lodInfo) {
+                    console.log(`[${NAME}] LOD ${i} doesnt exist`);
+                    continue;
+                }
+                const fullPath = outputDirectory + lodInfo.path;
+                if (existsSync(fullPath)) {
+                    const lastSize = prevSize;
+                    const sizeOnDisc = statSync(fullPath).size;
+                    const diff = lastSize - sizeOnDisc;
+                    const level = lodInfo.level;
+                    let shouldOmitLevel = false;
+                    // if the mesh size on disc is identical to the last level, we can omit this level
+                    if (sizeOnDisc <= 0) {
+                        console.log(`[${NAME}] Flag mesh LOD ${level}: size on disc is 0 → level will be omitted (if this mesh is is taken from cache it can mean that the LOD level was omitted before). \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
+                        shouldOmitLevel = true;
+                    }
+                    else if (diff == 0) {
+                        console.log(`[${NAME}] Flag mesh LOD ${level}: size on disc did not change compared to previous level → level will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
+                        shouldOmitLevel = true;
+                    }
+                    else if (!shouldOmitLevel) {
+                        // if the mesh vertex count difference to the last valid level is less than 32 vertices or less than 5% of the last level, we can omit this level
+                        const meshInfo = lodInfo.lodMeshInfo;
+                        const lastMeshInfo = generatedLODInfos[i - 1]?.lodMeshInfo;
+                        if (meshInfo && lastMeshInfo) {
+                            const vertexDifference = lastMeshInfo.totalVertexCount - meshInfo.totalVertexCount;
+                            const diff01 = vertexDifference / lastMeshInfo.totalVertexCount;
+                            if (vertexDifference < 0) {
+                                console.log(`WARN: ❌ [${NAME}] Flag mesh LOD ${level}: vertex count did increase (${-vertexDifference.toFixed(0)} vertices, ${(-diff01 * 100).toFixed(4)}%) → LOD ${level} will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\" ${JSON.stringify(lastMeshInfo)} → ${JSON.stringify(meshInfo)}`);
+                                shouldOmitLevel = true;
+                            }
+                            else if (vertexDifference < 32 || diff01 < 0.05) {
+                                console.log(`[${NAME}] Flag mesh LOD ${level}: vertex count difference below threshold (removed ${vertexDifference.toFixed(0)} vertices, ${(-diff01 * 100).toFixed(4)}%) → LOD ${level} will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
+                                shouldOmitLevel = true;
+                            }
+                        }
+                        if (shouldOmitLevel && opts.useCache != false && lodInfo.cacheKey != undefined) {
+                            // we invalidate the cached entry because the mesh change was too small
+                            addToCache(lodInfo.cacheKey, new Uint8Array(0));
+                        }
+                    }
+                    if (shouldOmitLevel) {
+                        generatedLODInfos.splice(i, 1);
+                        rmSync(fullPath);
+                        i--;
+                    }
+                    else {
+                        prevSize = sizeOnDisc;
+                    }
                 }
             }
+            let maxDensity = 0;
+            let primitiveIndex = 0;
+            const lodLevelInFile = generatedLODInfos.length;
             for (const prim of mesh.listPrimitives()) {
-                const lastLod = LODS[LODS.length - 1];
-                console.log(`${NAME}: ${filename} LOD ${LODS.length - 1}`);
-                simplifyMesh(doc, mesh, prim, lastLod);
+                const lastLod = LODS[lodLevelInFile];
+                // TODO: this should be run after simplification but then we need to change the runtime to use the LOD 0 density
+                const density = calculateMeshDensity(prim);
+                maxDensity = Math.max(maxDensity, density);
+                const task = simplifyMesh(doc, mesh, prim, primitiveIndex++, lastLod);
+                tasks.push(task);
             }
-            const model = progressiveExt.create({
-                guid: settings.guid,
-                uri: generatedLODs.map(l => l),
-            });
-            mesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, model);
-            index++;
-            if (!visited.has(settings.guid)) {
-                visited.set(settings.guid, {
-                    model: model,
-                    paths: generatedLODs,
+            await Promise.all(tasks);
+            tasks.length = 0;
+            const { totalVertexCount: totalVertexCountAfter } = getMeshInformation(mesh);
+            const difference = originalTotalVertexCount - totalVertexCountAfter;
+            const diff01 = difference / originalTotalVertexCount;
+            if (totalVertexCountAfter <= 0) {
+                logger.error(`ERR: [${NAME}]: Mesh[${index}] has no vertices after simplification: ${mesh.getName()}`);
+            }
+            else if (difference < 32 || diff01 <= 0.05) {
+                // the lowest LOD has the same vertex count as the original mesh → we don't need to write the extension and can also delete previously generated LODs
+                console.log(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length}: vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove LODs. \"${mesh.getName()}\"`);
+                try {
+                    for (const lodInfo of generatedLODInfos) {
+                        rmSync(outputDirectory + "/" + lodInfo.path);
+                    }
+                }
+                catch (err) {
+                    logger.error(`ERR: [${NAME}] Failed to remove LOD files for \"${mesh.getName()}\"`);
+                }
+                continue;
+            }
+            if (generatedLODInfos.length > 0) {
+                const model = progressiveExt.create({
+                    guid: settings.guid,
+                    lods: generatedLODInfos.map(l => {
+                        return {
+                            path: l.path,
+                            density: l.density,
+                        };
+                    }),
+                    density: maxDensity,
                 });
+                mesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, model);
+                index++;
+                if (!visited.has(settings.guid)) {
+                    visited.set(settings.guid, {
+                        model: model,
+                        infos: generatedLODInfos,
+                    });
+                }
             }
         }
-        console.log("< Progressive Mesh Transform: finished");
+        console.log(`[${NAME}]: Finished generating progressive meshes for \"${filename}\"`);
     });
 }
 /**
@@ -371,19 +509,47 @@ export function make_progressive_meshes(filePath, _options, opts) {
  * @param guid a unique identifier for the mesh
  * @param error the error to use for simplification. Higher means more simplification
  */
-function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
+async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLevel, opts) {
+    const fileExtension = ".glb";
+    const fileName_withoutExtension = "mesh_lod_" + level + "_" + guid;
+    const fileName = fileName_withoutExtension + fileExtension;
+    const outPath = basePath + fileName;
+    const primitives = mesh.listPrimitives();
+    let cacheKey = undefined;
+    if (opts.useCache !== false) {
+        // generate hash for mesh
+        let hash = mesh["needle:mesh-hash"];
+        if (hash === undefined) {
+            hash = getHash(mesh);
+            mesh["needle:mesh-hash"] = hash;
+        }
+        hash += getHash(lodLevel);
+        cacheKey = `mesh_lod_${level}_${hash}`;
+        const cachedValue = tryGetFromCache(cacheKey);
+        if (cachedValue) {
+            console.log(`[${NAME}] LOD ${level} loaded ${(cachedValue.length / 1024).toFixed()} KB from cache for \"${mesh.getName()}\"`);
+            writeFileSync(outPath, cachedValue);
+            return { level, path: fileName, success: true, cacheKey: cacheKey };
+        }
+    }
     const newDoc = new Document();
     const newScene = newDoc.createScene();
     newDoc.createBuffer();
     const newNode = newDoc.createNode();
     newScene.addChild(newNode);
-    newNode.setName("node_" + level + "_" + guid);
+    newNode.setName("LOD " + level + "_" + guid);
     newNode.setTranslation([0, 0, 0]);
     const newMesh = newDoc.createMesh();
+    newMesh.setName(mesh.getName() + " LOD " + level);
     newNode.setMesh(newMesh);
-    const primitives = mesh.listPrimitives();
+    const newMat = newDoc.createMaterial();
+    const weights = mesh.getWeights();
+    if (weights) {
+        newMesh.setWeights(weights);
+    }
+    const tasks = new Array();
+    // let didRunSimplification = false;
     for (const prim of primitives) {
-        const newMat = newDoc.createMaterial();
         const newPrim = newDoc.createPrimitive();
         newMesh.addPrimitive(newPrim);
         newPrim.setName(prim.getName());
@@ -398,6 +564,9 @@ function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
                 newAttr.setType(attr.getType());
                 newPrim.setAttribute(semantics[i], newAttr);
             }
+            else {
+                console.warn(`[${NAME}] Missing attribute ${semantics[i]} in mesh LOD ${level}`);
+            }
         }
         const indices = prim.getIndices();
         if (indices) {
@@ -406,21 +575,52 @@ function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
             newIndices.setType(indices.getType());
             newPrim.setIndices(newIndices);
         }
+        const targets = prim.listTargets();
+        for (const target of targets) {
+            const newTarget = newDoc.createPrimitiveTarget(target.getName());
+            newPrim.addTarget(newTarget);
+            for (const attr of target.listSemantics()) {
+                const accessor = target.getAttribute(attr);
+                if (accessor) {
+                    const newAccessor = newDoc.createAccessor();
+                    newAccessor.setArray(accessor.getArray());
+                    newAccessor.setType(accessor.getType());
+                    newTarget.setAttribute(attr, newAccessor);
+                }
+            }
+        }
+        ;
         const extensions = prim.listExtensions();
         for (const ext of extensions) {
             const PropertyClass = ext.constructor;
             const newProp = new PropertyClass(newDoc.getGraph());
             const copy = newProp.copy(ext);
             const name = ext.extensionName;
-            console.log(`${NAME}: Copy primitive ${name} extension`);
             newPrim.setExtension(name, copy);
         }
     }
+    let primitiveIndex = 0;
     if (lodLevel.error != undefined || lodLevel.ratio != undefined) {
-        for (const prim of newMesh.listPrimitives()) {
-            simplifyMesh(newDoc, newMesh, prim, lodLevel);
+        if (lodLevel.error > 0 || lodLevel.ratio < 1) {
+            for (const newPrim of newMesh.listPrimitives()) {
+                // didRunSimplification = true;
+                const task = simplifyMesh(newDoc, newMesh, newPrim, primitiveIndex++, lodLevel);
+                tasks.push(task);
+            }
         }
     }
+    await Promise.all(tasks);
+    tasks.length = 0;
+    const newMeshInformation = getMeshInformation(newMesh);
+    if (newMeshInformation.totalVertexCount < 4) {
+        console.error(`[${NAME}] mesh LOD ${level} is has less than 4 vertices → skipping`);
+        return { level, path: fileName, success: false, cacheKey: cacheKey };
+    }
+    let density = 0;
+    for (const prim of newMesh.listPrimitives()) {
+        const d = calculateMeshDensity(prim);
+        density = Math.max(density, d);
+    }
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
     if (!progressiveExt || !progressiveExt.create) {
         console.error("Failed to create progressive extension");
@@ -428,6 +628,7 @@ function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
     }
     newMesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
         guid: guid,
+        density: density,
     }));
     // we need to compress the mesh with the same compression method as the original file
     const root = doc.getRoot();
@@ -435,36 +636,40 @@ function createMeshLod(doc, basePath, level, mesh, guid, lodLevel) {
     if (compressionExt) {
         const newExt = newDoc.createExtension(NEEDLE_mesh_compression);
         const newProp = newExt.create(compressionExt.model);
-        console.log(`${NAME}: Pass compression format to mesh LOD ${level} → ${JSON.stringify(compressionExt.model)}`);
+        console.log(`[${NAME}] Pass compression format to mesh LOD ${level} → ${JSON.stringify(compressionExt.model)}`);
         newDoc.getRoot().setExtension(NEEDLE_mesh_compression.EXTENSION_NAME, newProp);
     }
     const newIo = new NodeIO();
     registerExtensions(newIo);
-    const ext = ".glb";
-    const name = "mesh_lod_" + level + "_" + guid;
-    const filename = name + ext;
-    const outName = basePath + filename;
-    console.log(`${NAME}: Write mesh LOD ${level} to \"${filename}\" with: ${JSON.stringify(lodLevel)}`);
-    newIo.write(outName, newDoc);
-    return filename;
+    console.log(`[${NAME}] Write mesh LOD ${level} with to \"${fileName}\"`);
+    await newIo.write(outPath, newDoc);
+    // const hasChanged = meshInfo.maxVertexCount == newMeshInformation.maxVertexCount && meshInfo.totalVertexCount == newMeshInformation.totalVertexCount;
+    // add to cache
+    if (cacheKey != undefined) {
+        const bytes = readFileSync(outPath);
+        addToCache(cacheKey, bytes);
+    }
+    return { level, path: fileName, success: true, lodMeshInfo: newMeshInformation, cacheKey: cacheKey, density: density };
 }
-function simplifyMesh(doc, mesh, prim, lodinfo) {
+async function simplifyMesh(doc, mesh, prim, index, lodinfo) {
     const weldOptions = {
         ...WELD_DEFAULTS,
         overwrite: true,
+        ...lodinfo,
     };
     const simplifyOptions = {
-        ...SIMPLIFY_DEFAULTS, simplifier: MeshoptSimplifier,
+        ...SIMPLIFY_DEFAULTS,
+        ...lodinfo,
+        simplifier: MeshoptSimplifier,
     };
-    if (lodinfo.error != undefined)
-        simplifyOptions.error = lodinfo.error;
-    if (lodinfo.ratio != undefined)
-        simplifyOptions.ratio = lodinfo.ratio;
-    compressPrimitive(doc, mesh, prim, weldOptions, simplifyOptions);
+    return compressPrimitive(doc, mesh, prim, index, weldOptions, simplifyOptions);
 }
-function isMeshLOD(filepath) {
+export function isMeshLOD(filepath) {
     return filepath.includes("mesh_lod_");
 }
-function isImageLOD(filepath) {
+export function isImageLOD(filepath) {
     return filepath.includes("image_");
 }
+export function isLOD(filepath) {
+    return isMeshLOD(filepath) || isImageLOD(filepath);
+}

package/dist/utils/guid.d.ts

@@ -0,0 +1 @@
+export declare function generateGuid(key: string): any;

package/dist/utils/guid.js

@@ -0,0 +1,5 @@
+import { v5 as uuidv5 } from 'uuid';
+const NEEDLE_BUILD_PIPELINE_NAMESPACE = '208dcaa6-1398-4d42-a625-8f5fc1cc1ccf';
+export function generateGuid(key) {
+    return uuidv5(key, NEEDLE_BUILD_PIPELINE_NAMESPACE);
+}

package/dist/utils/index.d.ts

@@ -1,2 +1,4 @@
 export * from "./fileutils.js";
 export * from "./version.js";
+export * from "./mesh.js";
+export * from "./guid.js";

package/dist/utils/index.js

@@ -1,2 +1,4 @@
 export * from "./fileutils.js";
 export * from "./version.js";
+export * from "./mesh.js";
+export * from "./guid.js";

package/dist/utils/mesh.d.ts

@@ -0,0 +1,7 @@
+import { Mesh, Primitive } from "@gltf-transform/core";
+export declare type MeshInformation = {
+    readonly maxVertexCount: number;
+    readonly totalVertexCount: number;
+};
+export declare function getMeshInformation(mesh: Mesh): MeshInformation;
+export declare function calculateMeshDensity(prim: Primitive): number;

package/dist/utils/mesh.js

@@ -0,0 +1,86 @@
+export function getMeshInformation(mesh) {
+    let maxVertexCount = 0;
+    const totalVertexCount = mesh.listPrimitives().reduce((acc, prim) => {
+        const vertices = prim.getAttribute('POSITION')?.getCount() || 0;
+        maxVertexCount = Math.max(maxVertexCount, vertices);
+        return acc + vertices;
+    }, 0);
+    return {
+        maxVertexCount,
+        totalVertexCount,
+    };
+}
+export function calculateMeshDensity(prim) {
+    const bounds = {
+        min: [Infinity, Infinity, Infinity],
+        max: [-Infinity, -Infinity, -Infinity],
+    };
+    const position = prim.getAttribute('POSITION');
+    if (!position)
+        return -1;
+    let tempVec3 = [0, 0, 0];
+    for (let i = 0; i < position.getCount(); i++) {
+        tempVec3 = position.getElement(i, tempVec3);
+        expandBounds(tempVec3, bounds);
+    }
+    const indices = prim.getIndices();
+    let area = 0;
+    if (indices) {
+        // calculate surface area
+        const numIndices = indices.getCount();
+        const indexArray = indices.getArray();
+        const v0 = [0, 0, 0];
+        const v1 = [0, 0, 0];
+        const v2 = [0, 0, 0];
+        const e1 = [0, 0, 0];
+        const e2 = [0, 0, 0];
+        const boundSizeX = bounds.max[0] - bounds.min[0];
+        const boundSizeY = bounds.max[1] - bounds.min[1];
+        const boundSizeZ = bounds.max[2] - bounds.min[2];
+        const maxBounds = Math.max(boundSizeX, boundSizeY, boundSizeZ);
+        for (let i = 0; i < numIndices; i += 3) {
+            const i0 = indexArray[i + 0];
+            const i1 = indexArray[i + 1];
+            const i2 = indexArray[i + 2];
+            position.getElement(i0, v0);
+            position.getElement(i1, v1);
+            position.getElement(i2, v2);
+            e1[0] = v1[0] - v0[0];
+            e1[1] = v1[1] - v0[1];
+            e1[2] = v1[2] - v0[2];
+            e2[0] = v2[0] - v0[0];
+            e2[1] = v2[1] - v0[1];
+            e2[2] = v2[2] - v0[2];
+            // Normalize vertex distance in mesh bounds
+            e1[0] /= maxBounds;
+            e1[1] /= maxBounds;
+            e1[2] /= maxBounds;
+            e2[0] /= maxBounds;
+            e2[1] /= maxBounds;
+            e2[2] /= maxBounds;
+            // Cross product to get the area of the triangle
+            const cross = [
+                e1[1] * e2[2] - e1[2] * e2[1],
+                e1[2] * e2[0] - e1[0] * e2[2],
+                e1[0] * e2[1] - e1[1] * e2[0]
+            ];
+            // Add the area (half the magnitude of the cross product) to the total
+            area += Math.sqrt(cross[0] * cross[0] + cross[1] * cross[1] + cross[2] * cross[2]) / 2;
+        }
+    }
+    const count = indices.getCount() / 3;
+    const density = count / area;
+    // const size = [0, 0, 0] as vec3;
+    // size[0] = bounds.max[0] - bounds.min[0];
+    // size[1] = bounds.max[1] - bounds.min[1];
+    // size[2] = bounds.max[2] - bounds.min[2];
+    // const volume = size[0] * size[1] * size[2];
+    // const density = count / volume;
+    return density;
+    function expandBounds(point, target) {
+        for (let i = 0; i < 3; i++) {
+            target.min[i] = Math.min(point[i], target.min[i]);
+            target.max[i] = Math.max(point[i], target.max[i]);
+        }
+    }
+}

package/index.mjs

@@ -1,6 +1,7 @@
 
-
 import { spawnSync } from "child_process";
+import { getVersion } from "./dist/index.js";
+
 const workingDir = process.cwd();
 
 // Original invoked dir set by npm
@@ -28,6 +29,13 @@ function runWithArgs(cmd) {
     spawnSync(cmd + " " + commandline_args, defaultArgs);
 }
 
-
-runWithArgs("node make-progressive.mjs");
-runWithArgs("node pack-gltf.mjs");
+try {
+    const startTime = Date.now();
+    runWithArgs("node make-progressive.mjs");
+    runWithArgs("node pack-gltf.mjs");
+    const durationInMinutes = ((Date.now() - startTime) / 1000 / 60).toFixed(1);
+    console.log(`[Needle Build Pipeline] v${getVersion()} — Finished progressive transform and optimization in ${durationInMinutes} minutes.`);
+}
+catch (err) {
+    console.error("Failed running commandline args:", err.message);
+}

package/pack-gltf.mjs

@@ -1,20 +1,18 @@
-import glob from 'glob';
 import path from 'path';
 import { NodeIO } from '@gltf-transform/core';
 import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
-import { draco } from '@gltf-transform/functions';
 import draco3d from 'draco3dgltf';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS } from "./dist/extensions/NEEDLE_opaque/index.js";
 import commandLineArgs from "command-line-args";
 import { NEEDLE_mesh_compression, NEEDLE_compression_texture } from './dist/extensions/index.js';
-import { existsSync, statSync } from 'fs';
+import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { needle_mesh_transform } from "./dist/transforms/needle_mesh_transform.js";
 import { needle_texture_transform } from "./dist/transforms/needle_texture_transform.js";
 import { needle_animation_transform } from "./dist/transforms/needle_animation_transform.js";
 import { foreachGLTF } from './fileutils.mjs';
 import { needle_asset } from './dist/transforms/needle_asset.js';
 import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
-import { getVersion } from './dist/index.js';
+import { getVersion, getHash, addToCache, tryGetFromCache, isLOD, ensureCacheSize as ensureMaxCacheSize, isMeshLOD } from './dist/index.js';
 
 console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`)
 
@@ -77,6 +75,9 @@ let totalFilesProcessed = 0;
 let totalFileSizeInMB = 0;
 let totalFileSizeInMBBefore = 0;
 
+/** Compress a glTF file
+ * @param {string} file
+ */
 async function packFile(file) {
     if (!existsSync(file)) {
         console.error("WARN: can not pack, file does not exist: " + file);
@@ -86,6 +87,22 @@ async function packFile(file) {
         return;
     }
 
+    // check if it's in the cache. 
+    // We only need to check mesh LOD files because image LODs are cache via our toktx transform
+    // We can not cache the main glTF files because the hash always changes for them
+    let cacheKey = undefined;
+    if (allowCaching && isMeshLOD(file)) {
+        const existing = readFileSync(file);
+        const hash = getHash(existing);
+        cacheKey = "packed-" + hash;
+        const cached = tryGetFromCache(cacheKey);
+        if (cached) {
+            console.log(`→ Loaded compressed file from cache: \"${path.basename(file)}\"`);
+            writeFileSync(file, cached);
+            return;
+        }
+    }
+
     if (packedFiles.has(file)) return;
     packedFiles.add(file);
     console.log("→ Begin packing: " + file);
@@ -123,6 +140,10 @@ async function packFile(file) {
         );
         if (debug) file += ".debug.glb";
         await io.write(file, document);
+
+        if (allowCaching && cacheKey != undefined && existsSync(file))
+            addToCache(cacheKey, readFileSync(file));
+
         const duration = Date.now() - startTime;
         const sizeInMB = statSync(file).size / 1024 / 1024;
         const filename = path.basename(file);
@@ -148,8 +169,17 @@ let startTime = Date.now();
 const inputArgs = args.slice(2);
 foreachGLTF(inputArgs, async (file) => {
     await packFile(file);
-}).then(() => {
+}).then(async () => {
     const duration = Date.now() - startTime;
     const durationInMinutes = duration / 1000 / 60;
+    if (allowCaching) {
+        // if the user's cache directory has more than 25GB free space, we allow a cache of 2 GB - otherwise 1 GB
+        // const freeSpaceInMB = (await getCacheAvailableSpaceOnDisc().catch(() => 0)) / 1024 / 1024;
+        // const maxCacheSize = freeSpaceInMB > 25000 ? 2000 : 1000;
+        ensureMaxCacheSize(1000);
+    }
     console.log(`[Needle Build Pipeline] v${getVersion()} — Finished packing ${totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${totalFileSizeInMBBefore.toFixed(1)} MB → ${totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${totalGPUMemoryInMB.toFixed(1)} MB`);
-})
+}).catch(err => {
+    console.error("ERR: packing failed: " + err.message);
+    if (debug) throw err;
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.4.1-beta.1",
+  "version": "1.4.1-beta.11",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",
@@ -16,9 +16,9 @@
     "compile": "tsc || exit 1"
   },
   "dependencies": {
-    "@gltf-transform/core": "^3.10.0",
-    "@gltf-transform/extensions": "^3.10.0",
-    "@gltf-transform/functions": "^3.10.0",
+    "@gltf-transform/core": "^3.10.1",
+    "@gltf-transform/extensions": "^3.10.1",
+    "@gltf-transform/functions": "^3.10.1",
     "command-exists": "^1.2.9",
     "command-line-args": "^5.2.1",
     "draco3dgltf": "^1.5.3",
@@ -31,7 +31,8 @@
     "p-limit": "3.1.0",
     "semver": "^7.3.7",
     "sharp": "^0.31.3",
-    "tmp": "^0.2.1"
+    "tmp": "^0.2.1",
+    "uuid": "^9.0.1"
   },
   "watch": {
     "compile": {