@needle-tools/gltf-build-pipeline 1.4.1-beta.9 → 1.4.1

package/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.4.1] - 2023-04-02
+- release
+
+## [1.4.1-beta.15] - 2023-04-02
+- fix: issue where discarding all LODs might result in broken mesh due to simplify having modified the original mesh
+- fix: don't produce LODs if GLB/glTF was produced with old generator
+
+## [1.4.1-beta.13] - 2023-03-26
+- change: passthrough `NEEDLE_editor` & `NEEDLE_gltf_dependencies` extensions
+
+## [1.4.1-beta.12] - 2023-03-23
+- add: NEEDLE_progressive LODs can now contain more info per texture or mesh. Meshes contain vertex- and index counts
+- change: the cache can now store and retrieve meta information
+
+## [1.4.1-beta.11] - 2023-03-20
+- add: NEEDLE_progressive LOD meshes morph targets data
+- fix: NEEDLE_progressive should check if simplifcation did result in improved meshes (> 32 verts removed && > 5 % improvement) and otherwise discard LODs (and the whole extension if no improvement was made)
+- change: Unity NEEDLE_progressive logs
+
+## [1.4.1-beta.10] - 2023-03-19
+- change: NEEDLE_progressive calculates density per mesh level
+- change: NEEDLE_progressive `uri` field is removed and added `lod` array of objects
+
 ## [1.4.1-beta.8] - 2023-03-18
 - change: if an asset doesnt have a guid we use a uuid package now
 - change: ignore mesh LODs with vertex count < 4

package/dist/cache/cache.d.ts

@@ -1,10 +1,47 @@
 import { ExtensibleProperty } from "@gltf-transform/core";
 /**
- * Get the available space on the cache directory in bytes.
+ * Clear the cache.
+ * This will remove all files in the cache directory.
  */
 export declare function clearCache(): void;
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
 export declare function ensureCacheSize(maxSizeMB: number): void;
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
 export declare function addToCache(key: string, bytes: Uint8Array): void;
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
 export declare function tryGetFromCache(key: string): Uint8Array | null;
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export declare function addMetaToCache(key: string, meta: object): void;
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export declare function tryGetMetaFromCache(key: string, ignoreVersion?: boolean): object | null;
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
 export declare function getKey(property: ExtensibleProperty): string;
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @returns The hash as a number.
+ */
 export declare function getHash(property: ExtensibleProperty | object): number;

package/dist/cache/cache.js

@@ -2,6 +2,7 @@ import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-tr
 import tmp from 'tmp';
 import fs from 'fs';
 import path from 'path';
+import { getVersion } from "../utils/version.js";
 // import * as CHECKDISCSPACE from 'check-disk-space'
 /**
  * Get the available space on the cache directory in bytes.
@@ -17,11 +18,20 @@ import path from 'path';
 //     }
 //     return 0;
 // }
+const buildPipelineVersion = getVersion(false);
+/**
+ * Clear the cache.
+ * This will remove all files in the cache directory.
+ */
 export function clearCache() {
     const dir = getCacheDirectory();
     fs.rmSync(dir, { recursive: true });
     console.log(`Cleared cache at ${dir}`);
 }
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
 export function ensureCacheSize(maxSizeMB) {
     const dir = getCacheDirectory();
     let totalSize = 0;
@@ -50,12 +60,23 @@ export function ensureCacheSize(maxSizeMB) {
     }
     console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
 }
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
 export function addToCache(key, bytes) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
     console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
     fs.writeFileSync(path, bytes);
 }
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
 export function tryGetFromCache(key) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
@@ -65,6 +86,50 @@ export function tryGetFromCache(key) {
     }
     return null;
 }
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export function addMetaToCache(key, meta) {
+    const dir = getCacheDirectory();
+    const metaPath = `${dir}/${key}.meta`;
+    meta['needle-build-pipeline:version'] = buildPipelineVersion;
+    fs.writeFileSync(metaPath, JSON.stringify(meta));
+}
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export function tryGetMetaFromCache(key, ignoreVersion = false) {
+    try {
+        const dir = getCacheDirectory();
+        const path = `${dir}/${key}.meta`;
+        if (fs.existsSync(path)) {
+            const bytes = fs.readFileSync(path);
+            if (bytes?.length > 0) {
+                const text = bytes.toString('utf-8');
+                if (text) {
+                    const obj = JSON.parse(text);
+                    if (ignoreVersion)
+                        return obj;
+                    if (obj['needle-build-pipeline:version'] === buildPipelineVersion)
+                        return obj;
+                }
+            }
+        }
+        return null;
+    }
+    catch (e) {
+        console.error(`Error reading cache meta: ${e.message}`);
+        return null;
+    }
+}
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
 export function getKey(property) {
     let prefix = "prop";
     if (property instanceof Texture) {
@@ -74,6 +139,11 @@ export function getKey(property) {
     const key = `${prefix}-${hash}`;
     return key;
 }
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @returns The hash as a number.
+ */
 export function getHash(property) {
     let hash = 0;
     if (property instanceof ExtensibleProperty) {

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts

@@ -20,5 +20,6 @@ export interface IExtensionWriter {
     assignTo?(obj: object): any;
     write(context: WriterContext, prop: IExtensibleProperty): void;
 }
+export declare let currentGenerator: string;
 export declare function createOpaqueExtension(name: string, types: PropertyType | PropertyType[], opts?: Options): IOpaqueExtension;
 export {};

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -4,6 +4,7 @@ const ALL_PROPERTY_TYPES = [];
 for (const key in PropertyType) {
     ALL_PROPERTY_TYPES.push(PropertyType[key]);
 }
+export let currentGenerator = "";
 // an opaque extension can be used to pass data through the gltf transform process
 // it should also be able to re-write/fix json pointers after the transformation which might have shuffled them around 
 export function createOpaqueExtension(name, types, opts) {
@@ -18,6 +19,7 @@ export function createOpaqueExtension(name, types, opts) {
         // private roots: Array<OpaqueProperty> = [];
         jsonPointerHandler;
         read(context) {
+            currentGenerator = context.jsonDoc.json.asset?.generator;
             if (debugLog)
                 console.log("READ " + EXTENSION_NAME);
             this.jsonPointerHandler = new JsonPointerHandler(this.document, EXTENSION_NAME);

package/dist/extensions/NEEDLE_opaque/index.js

@@ -15,4 +15,6 @@ export const ALL_EXTENSIONS = [
     createOpaqueExtension("NEEDLE_render_objects", PropertyType.ROOT, { ...opaqueOptions }),
     createOpaqueExtension("NEEDLE_progressive", [PropertyType.TEXTURE, PropertyType.MATERIAL, PropertyType.MESH], { ...opaqueOptions }),
     createOpaqueExtension("EXT_texture_exr", [PropertyType.TEXTURE], { ...opaqueOptions }),
+    createOpaqueExtension("NEEDLE_editor", [PropertyType.NODE, PropertyType.MATERIAL], { ...opaqueOptions }),
+    createOpaqueExtension("NEEDLE_gltf_dependencies", [PropertyType.ROOT], { ...opaqueOptions }),
 ];

package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.d.ts

@@ -1,25 +1,39 @@
-import { Extension, ExtensionProperty, Graph, IProperty, Property, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
+import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare const NEEDLE_PROGRESSIVE_EXTENSION_NAME = "NEEDLE_progressive";
-export declare interface IProgressiveLoadingInfo {
+export type ProgressiveLoadingInfo = {
     /** guid of the asset the LODs belong to */
     guid: string;
-    /** paths to the LOD level files */
-    uri?: string | Array<string>;
-    density?: number;
-}
+    /** lod infos and paths */
+    lods?: Array<LodInfo>;
+};
+type LodInfo = {
+    path: string;
+};
+export type ProgressiveTextureModel = ProgressiveLoadingInfo & {
+    lods?: Array<LodInfo & {}>;
+};
+export type ProgressiveMeshModel = ProgressiveLoadingInfo & {
+    density: number;
+    lods?: Array<LodInfo & {
+        density: number;
+        vertexCount: number;
+        indexCount: number;
+    }>;
+};
 export declare class NEEDLE_progressive extends Extension {
     static EXTENSION_NAME: string;
     extensionName: string;
-    static create(graph: Graph<Property<IProperty>>, data: IProgressiveLoadingInfo): NEEDLE_progressive_texture_property;
-    create(data: IProgressiveLoadingInfo): NEEDLE_progressive_texture_property;
+    createTexture(data: ProgressiveTextureModel): NEEDLE_progressive_property;
+    createMesh(data: ProgressiveMeshModel): NEEDLE_progressive_property;
     read(_context: ReaderContext): this;
     write(context: WriterContext): this;
 }
-export declare class NEEDLE_progressive_texture_property extends ExtensionProperty {
+export declare class NEEDLE_progressive_property extends ExtensionProperty {
     static EXTENSION_NAME: string;
     readonly extensionName: string;
     parentTypes: PropertyType[];
     propertyType: string;
     init(): void;
-    model: IProgressiveLoadingInfo;
+    model: ProgressiveLoadingInfo;
 }
+export {};

package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.js

@@ -4,13 +4,15 @@ export const NEEDLE_PROGRESSIVE_EXTENSION_NAME = "NEEDLE_progressive";
 export class NEEDLE_progressive extends Extension {
     static EXTENSION_NAME = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     extensionName = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
-    static create(graph, data) {
-        const ext = new NEEDLE_progressive_texture_property(graph);
+    createTexture(data) {
+        const ext = new NEEDLE_progressive_property(this.document.getGraph());
         ext.model = data;
         return ext;
     }
-    create(data) {
-        return NEEDLE_progressive.create(this.document.getGraph(), data);
+    createMesh(data) {
+        const ext = new NEEDLE_progressive_property(this.document.getGraph());
+        ext.model = data;
+        return ext;
     }
     read(_context) {
         // console.log("READ " + NEEDLE_PROGRESSIVE_TEXTURE_EXTENSION_NAME);
@@ -57,7 +59,7 @@ export class NEEDLE_progressive extends Extension {
         return this;
     }
 }
-export class NEEDLE_progressive_texture_property extends ExtensionProperty {
+export class NEEDLE_progressive_property extends ExtensionProperty {
     static EXTENSION_NAME = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     extensionName = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     parentTypes = [PropertyType.TEXTURE, PropertyType.MESH];

package/dist/transforms/needle_mesh_transform.d.ts

@@ -1,4 +1,4 @@
 import { Transform, Document, Primitive, Mesh } from "@gltf-transform/core";
 import { SimplifyOptions, WeldOptions } from "@gltf-transform/functions";
 export declare const needle_mesh_transform: (_options: SimplifyOptions) => Transform;
-export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<void>;
+export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, index: number, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<Primitive>;

package/dist/transforms/needle_mesh_transform.js

@@ -50,7 +50,9 @@ async function simplifierTransform(document, _context, _options) {
         overwrite: true,
     };
     for (const mesh of document.getRoot().listMeshes()) {
+        let i = 0;
         for (const prim of mesh.listPrimitives()) {
+            const index = i++;
             // TODO: mesh compression caching
             // const key = getKey(prim);
             // const cached = tryGetFromCache(key);
@@ -84,14 +86,14 @@ async function simplifierTransform(document, _context, _options) {
                 logger.warn(`[${NAME}]: Skipping primitive of mesh "${mesh.getName()}": Requires TRIANGLES draw mode.`);
                 continue;
             }
-            compressPrimitive(document, mesh, prim, weldOptions, simplifyOpts);
+            compressPrimitive(document, mesh, prim, index, weldOptions, simplifyOpts);
         }
     }
     return true;
 }
-export async function compressPrimitive(document, mesh, prim, weldOptions, meshOptions) {
+export async function compressPrimitive(document, mesh, prim, index, weldOptions, meshOptions) {
     const logger = document.getLogger();
-    logger.debug(`[${NAME}]: Welding primitive of mesh "${mesh.getName()}" → tolerance: ${weldOptions.tolerance}`);
+    logger.debug(`weld: primitive[${index}] of mesh "${mesh.getName()}" → tolerance: ${weldOptions.tolerance}`);
     weldPrimitive(document, prim, weldOptions);
     // make sure we wait for the mesh simplifier to be ready
     if (meshOptions.simplifier && meshOptions.simplifier?.ready instanceof Promise) {
@@ -107,8 +109,8 @@ export async function compressPrimitive(document, mesh, prim, weldOptions, meshO
         if (!res)
             return;
     }
-    logger.debug(`[${NAME}]: Simplifying primitive of mesh "${mesh.getName()}" → ratio: ${meshOptions.ratio}, error: ${meshOptions.error}, lockBorder: ${meshOptions.lockBorder}`);
-    simplifyPrimitive(document, prim, meshOptions);
+    logger.debug(`simplify: primitive[${index}] of mesh "${mesh.getName()}" → ratio: ${meshOptions.ratio}, error: ${meshOptions.error}, lockBorder: ${meshOptions.lockBorder}`);
+    return simplifyPrimitive(document, prim, meshOptions);
 }
 // function updatePrimitive(prim: Primitive, simplifier: any, positionArray, dstIndicesArray) {
 // 	const position = prim.getAttribute('POSITION')!;

package/dist/transforms/needle_progressive.js

@@ -1,10 +1,10 @@
 import { BufferUtils, Document, NodeIO, PropertyType } from '@gltf-transform/core';
 import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots, sparse } from '@gltf-transform/functions';
 import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive } from "../extensions/NEEDLE_progressive/index.js";
-import { createOpaqueExtension, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
+import { createOpaqueExtension, currentGenerator, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
 import path from 'path';
 import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
-import { addToCache, getHash, getKey, tryGetFromCache } from '../cache/index.js';
+import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
 import { foreachGLTF, generateGuid } from '../utils/index.js';
 import { needle_asset } from './needle_asset.js';
@@ -139,7 +139,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 continue;
             }
             const progressiveExt = doc.createExtension(NEEDLE_progressive);
-            if (!progressiveExt || !progressiveExt.create) {
+            if (!progressiveExt || !progressiveExt.createTexture) {
                 logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${textureIndex}`);
                 continue;
             }
@@ -150,9 +150,13 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             }
             const relName = path.relative(directory, outPath);
             // TODO: this path should be added to the NEEDLE dependencies extension
-            texture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
+            texture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createTexture({
                 guid: textureGuid,
-                uri: relName,
+                lods: [
+                    {
+                        path: relName,
+                    }
+                ],
             }));
             const key = getKey(texture);
             if (opts.useCache) {
@@ -227,7 +231,7 @@ function createTextureLod(basePath, level, texture, slots, guid, textureIndex) {
     }
     // add progressive texture extension
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
-    newTexture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
+    newTexture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createTexture({
         guid: guid,
     }));
     const newIo = new NodeIO();
@@ -295,6 +299,12 @@ export function make_progressive_meshes(filePath, _options, opts) {
     // const fileNameWithoutExtension = path.basename(filePath, path.extname(filePath));
     const outputDirectory = directory + "/";
     return createTransform("PROGRESSIVE_MESHES", async (doc, _context) => {
+        // this is to ensure we're not creating progressive meshes for engine versions that don't support loading them yet
+        // the generator should include the version or the app that produced the GLB too (needle engine repo: 2c00d0e304e205cf9d4ac0499faabb92a706a5c4)
+        if (currentGenerator == "Needle Engine") {
+            console.log("WARN: Skip creating mesh LODs - Unsupported generator found... Please update your editor integration");
+            return;
+        }
         if (isMeshLOD(filePath)) {
             console.log("Progressive Mesh Transform: skip lod at " + filePath);
             return Promise.resolve();
@@ -302,7 +312,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
         await MeshoptSimplifier.ready;
         const logger = doc.getLogger();
         const filename = path.basename(filePath);
-        console.log("> Progressive Mesh Transform: " + filename);
+        console.log(`[${NAME}]: Begin generating progressive meshes for \"${filename}\"`);
         let index = 0;
         // the lowest LOD (last entry) will be saved in the original file
         const MAX_LODS = 6;
@@ -327,32 +337,38 @@ export function make_progressive_meshes(filePath, _options, opts) {
         }
         /** Map of visited GUID -> extension */
         const visited = new Map();
-        const meshes = doc.getRoot().listMeshes();
+        const root = doc.getRoot();
+        const meshes = root.listMeshes();
         for (const mesh of meshes) {
-            const meshInformation = getMeshInformation(mesh);
-            const { totalVertexCount, maxVertexCount } = meshInformation;
+            await processMesh(mesh);
+        }
+        console.log(`[${NAME}]: Finished generating progressive meshes for \"${filename}\"`);
+        async function processMesh(mesh) {
+            /** The size and vertex count of the original mesh */
+            const originalMeshInformation = getMeshInformation(mesh);
+            const { totalVertexCount: originalTotalVertexCount, maxVertexCount: originalMaxVertexCount } = originalMeshInformation;
             let settings = mesh.getExtension(NEEDLE_progressive_mesh_settings.EXTENSION_NAME);
             if (!settings) {
-                if (maxVertexCount < 32) {
-                    logger.debug(`[${NAME}] Skipping, no settings found and no primitive with more than 32 vertices. mesh[${index}] has ${totalVertexCount} vertices (max: ${maxVertexCount})`);
-                    continue;
+                if (originalMaxVertexCount < 32) {
+                    logger.debug(`[${NAME}] Skipping, no settings found and no primitive with more than 32 vertices. mesh[${index}] has ${originalTotalVertexCount} vertices (max: ${originalMaxVertexCount})`);
+                    return;
                 }
                 else {
                     settings = {
                         guid: generateGuid(path + "_mesh_" + index.toString()),
                         generateLods: true,
                     };
-                    logger.debug(`[${NAME}] No settings found for mesh[${index}] with ${totalVertexCount} vertices (max: ${maxVertexCount}) - will use default settings (guid: ${settings.guid})`);
+                    logger.debug(`[${NAME}] No settings found for mesh[${index}] with ${originalTotalVertexCount} vertices (max: ${originalMaxVertexCount}) - will use default settings (guid: ${settings.guid})`);
                 }
             }
             else if (settings.generateLods === false) {
                 logger.debug(`[${NAME}] progressive mesh disabled for mesh at index ${index++}`);
-                continue;
+                return;
             }
             const progressiveExt = doc.createExtension(NEEDLE_progressive);
-            if (!progressiveExt || !progressiveExt.create) {
+            if (!progressiveExt || !progressiveExt.createMesh) {
                 logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${index++}`);
-                continue;
+                return;
             }
             // check if we have already produced LODs for this mesh
             let needsToGenerateLODs = true;
@@ -367,12 +383,13 @@ export function make_progressive_meshes(filePath, _options, opts) {
             /** async simplification tasks... */
             const tasks = new Array();
             if (needsToGenerateLODs) {
+                console.log(`[${NAME}] Begin generating mesh LODs for \"${mesh.getName()}\"`);
                 // produce LODS, skip the last level because it will be saved in the original file
                 for (let level = 0; level < LODS.length - 1; level++) {
                     const lod = LODS[level];
                     const guid = settings.guid;
                     // TODO: check if the LOD level did actually change the mesh - otherwise we can just skip generating lower resolutions
-                    const lodTask = createMeshLod(doc, outputDirectory, level, mesh, meshInformation, guid, lod, opts);
+                    const lodTask = createMeshLod(doc, outputDirectory, level, mesh, originalMeshInformation, guid, lod, opts);
                     lodTask.then(res => {
                         if (res?.success) {
                             generatedLODInfos[level] = res;
@@ -402,11 +419,11 @@ export function make_progressive_meshes(filePath, _options, opts) {
                     let shouldOmitLevel = false;
                     // if the mesh size on disc is identical to the last level, we can omit this level
                     if (sizeOnDisc <= 0) {
-                        console.log(`[${NAME}] mesh LOD ${level} size on disc is 0 - level will be omitted (if this mesh is is taken from cache it can mean that the LOD level was omitted before) \"${lodInfo.path}\"`);
+                        console.log(`[${NAME}] Flag mesh LOD ${level}: size on disc is 0 → level will be omitted (if this mesh is is taken from cache it can mean that the LOD level was omitted before). \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                         shouldOmitLevel = true;
                     }
                     else if (diff == 0) {
-                        console.log(`[${NAME}] mesh LOD ${level} size on disc did not change compared to previous level - level will be omitted \"${lodInfo.path}\"`);
+                        console.log(`[${NAME}] Flag mesh LOD ${level}: size on disc did not change compared to previous level → level will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                         shouldOmitLevel = true;
                     }
                     else if (!shouldOmitLevel) {
@@ -415,17 +432,17 @@ export function make_progressive_meshes(filePath, _options, opts) {
                         const lastMeshInfo = generatedLODInfos[i - 1]?.lodMeshInfo;
                         if (meshInfo && lastMeshInfo) {
                             const vertexDifference = lastMeshInfo.totalVertexCount - meshInfo.totalVertexCount;
-                            const diffPercent = vertexDifference / lastMeshInfo.totalVertexCount;
+                            const diff01 = vertexDifference / lastMeshInfo.totalVertexCount;
                             if (vertexDifference < 0) {
-                                console.log(`WARN: [${NAME}] mesh LOD ${level} did increase vertex count (${-vertexDifference.toFixed(0)} vertices, ${(-diffPercent * 100).toFixed(4)}%) - level will be omitted \"${lodInfo.path}\" ${JSON.stringify(lastMeshInfo)} → ${JSON.stringify(meshInfo)}`);
+                                console.log(`WARN: ❌ [${NAME}] Flag mesh LOD ${level}: vertex count did increase (${-vertexDifference.toFixed(0)} vertices, ${(-diff01 * 100).toFixed(4)}%) → LOD ${level} will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\" ${JSON.stringify(lastMeshInfo)} → ${JSON.stringify(meshInfo)}`);
                                 shouldOmitLevel = true;
                             }
-                            else if (vertexDifference < 32 || diffPercent < 0.05) {
-                                console.log(`[${NAME}] mesh LOD ${level} change was too low (${-vertexDifference.toFixed(0)} vertices, ${(-diffPercent * 100).toFixed(4)}%) - level will be omitted \"${lodInfo.path}\" ${JSON.stringify(lastMeshInfo)} → ${JSON.stringify(meshInfo)}`);
+                            else if (vertexDifference < 32 || diff01 < 0.05) {
+                                console.log(`[${NAME}] Flag mesh LOD ${level}: vertex count difference below threshold (removed ${vertexDifference.toFixed(0)} vertices, ${(-diff01 * 100).toFixed(4)}%) → LOD ${level} will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                                 shouldOmitLevel = true;
                             }
                         }
-                        if (shouldOmitLevel) {
+                        if (shouldOmitLevel && opts.useCache != false && lodInfo.cacheKey != undefined) {
                             // we invalidate the cached entry because the mesh change was too small
                             addToCache(lodInfo.cacheKey, new Uint8Array(0));
                         }
@@ -441,25 +458,59 @@ export function make_progressive_meshes(filePath, _options, opts) {
                 }
             }
             let maxDensity = 0;
+            let primitiveIndex = 0;
+            const lodLevelInFile = generatedLODInfos.length;
+            const clonedData = new Array();
             for (const prim of mesh.listPrimitives()) {
-                const lodsCount = generatedLODInfos.length;
-                const lastLod = LODS[lodsCount];
-                console.log(`[${NAME}] ${mesh.getName()} LOD ${lodsCount}`);
+                const data = clonePrimitiveData(prim);
+                clonedData.push(data);
+                console.log(`[${NAME}] Simplify mesh[${index}] primitive[${primitiveIndex}] with ${data.indices.length / 3} triangles`);
+                const lastLod = LODS[lodLevelInFile];
+                // TODO: this should be run after simplification but then we need to change the runtime to use the LOD 0 density
                 const density = calculateMeshDensity(prim);
                 maxDensity = Math.max(maxDensity, density);
-                const task = simplifyMesh(doc, mesh, prim, lastLod);
+                const task = simplifyMesh(doc, mesh, prim, primitiveIndex++, lastLod);
                 tasks.push(task);
             }
             await Promise.all(tasks);
             tasks.length = 0;
             const { totalVertexCount: totalVertexCountAfter } = getMeshInformation(mesh);
+            const difference = originalTotalVertexCount - totalVertexCountAfter;
+            const diff01 = difference / originalTotalVertexCount;
             if (totalVertexCountAfter <= 0) {
-                logger.error(`ERR: [${NAME}] Mesh[${index}] has no vertices after simplification: ${mesh.getName()}`);
+                logger.error(`ERR: [${NAME}]: Mesh[${index}] has no vertices after simplification: ${mesh.getName()}`);
+            }
+            else if (difference < 32 || diff01 <= 0.05) {
+                // the lowest LOD has the same vertex count as the original mesh → we don't need to write the extension and can also delete previously generated LODs
+                console.log(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length}: vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove LODs. \"${mesh.getName()}\"`);
+                try {
+                    for (const lodInfo of generatedLODInfos) {
+                        rmSync(outputDirectory + "/" + lodInfo.path);
+                    }
+                }
+                catch (err) {
+                    logger.error(`ERR: [${NAME}] Failed to remove LOD files for \"${mesh.getName()}\"`);
+                }
+                // reset data (simplify might have changed the mesh data that we cloned before)
+                const primitives = mesh.listPrimitives();
+                if (clonedData.length == primitives.length) {
+                    for (let i = 0; i < primitives.length; i++) {
+                        setPrimitiveData(primitives[i], clonedData[i]);
+                    }
+                }
+                return;
             }
             if (generatedLODInfos.length > 0) {
-                const model = progressiveExt.create({
+                const model = progressiveExt.createMesh({
                     guid: settings.guid,
-                    uri: generatedLODInfos.map(l => l.path),
+                    lods: generatedLODInfos.map(l => {
+                        return {
+                            path: l.path,
+                            density: l.density,
+                            vertexCount: l.lodMeshInfo?.totalVertexCount,
+                            indexCount: l.lodMeshInfo?.totalIndexCount,
+                        };
+                    }),
                     density: maxDensity,
                 });
                 mesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, model);
@@ -472,9 +523,31 @@ export function make_progressive_meshes(filePath, _options, opts) {
                 }
             }
         }
-        console.log("< Progressive Mesh Transform: finished");
     });
 }
+function clonePrimitiveData(prim) {
+    const indices = new Uint32Array(prim.getIndices().getArray());
+    const attributes = {};
+    for (const attr of prim.listSemantics()) {
+        const accessor = prim.getAttribute(attr);
+        if (accessor) {
+            attributes[attr] = new Float32Array(accessor.getArray());
+        }
+    }
+    return { indices, attributes };
+}
+function setPrimitiveData(prim, data) {
+    const indices = prim.getIndices();
+    if (indices) {
+        indices.setArray(data.indices);
+    }
+    for (const attr of prim.listSemantics()) {
+        const accessor = prim.getAttribute(attr);
+        if (accessor) {
+            accessor.setArray(data.attributes[attr]);
+        }
+    }
+}
 /**
  * @param basePath the base path to save the lod files
  * @param level: 0 = original, 1 = first lod, 2 = second lod, ...
@@ -499,10 +572,11 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
         hash += getHash(lodLevel);
         cacheKey = `mesh_lod_${level}_${hash}`;
         const cachedValue = tryGetFromCache(cacheKey);
-        if (cachedValue) {
+        const meta = tryGetMetaFromCache(cacheKey);
+        if (cachedValue && meta) {
             console.log(`[${NAME}] LOD ${level} loaded ${(cachedValue.length / 1024).toFixed()} KB from cache for \"${mesh.getName()}\"`);
             writeFileSync(outPath, cachedValue);
-            return { level, path: fileName, success: true, cacheKey: cacheKey };
+            return { level, path: fileName, success: true, cacheKey: cacheKey, lodMeshInfo: meta.meshInfo, density: meta.density };
         }
     }
     const newDoc = new Document();
@@ -516,6 +590,10 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     newMesh.setName(mesh.getName() + " LOD " + level);
     newNode.setMesh(newMesh);
     const newMat = newDoc.createMaterial();
+    const weights = mesh.getWeights();
+    if (weights) {
+        newMesh.setWeights(weights);
+    }
     const tasks = new Array();
     // let didRunSimplification = false;
     for (const prim of primitives) {
@@ -533,6 +611,9 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
                 newAttr.setType(attr.getType());
                 newPrim.setAttribute(semantics[i], newAttr);
             }
+            else {
+                console.warn(`[${NAME}] Missing attribute ${semantics[i]} in mesh LOD ${level}`);
+            }
         }
         const indices = prim.getIndices();
         if (indices) {
@@ -541,6 +622,21 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
             newIndices.setType(indices.getType());
             newPrim.setIndices(newIndices);
         }
+        const targets = prim.listTargets();
+        for (const target of targets) {
+            const newTarget = newDoc.createPrimitiveTarget(target.getName());
+            newPrim.addTarget(newTarget);
+            for (const attr of target.listSemantics()) {
+                const accessor = target.getAttribute(attr);
+                if (accessor) {
+                    const newAccessor = newDoc.createAccessor();
+                    newAccessor.setArray(accessor.getArray());
+                    newAccessor.setType(accessor.getType());
+                    newTarget.setAttribute(attr, newAccessor);
+                }
+            }
+        }
+        ;
         const extensions = prim.listExtensions();
         for (const ext of extensions) {
             const PropertyClass = ext.constructor;
@@ -550,11 +646,12 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
             newPrim.setExtension(name, copy);
         }
     }
-    for (const newPrim of newMesh.listPrimitives()) {
-        if (lodLevel.error != undefined || lodLevel.ratio != undefined) {
-            if (lodLevel.error > 0 || lodLevel.ratio < 1) {
+    let primitiveIndex = 0;
+    if (lodLevel.error != undefined || lodLevel.ratio != undefined) {
+        if (lodLevel.error > 0 || lodLevel.ratio < 1) {
+            for (const newPrim of newMesh.listPrimitives()) {
                 // didRunSimplification = true;
-                const task = simplifyMesh(newDoc, newMesh, newPrim, lodLevel);
+                const task = simplifyMesh(newDoc, newMesh, newPrim, primitiveIndex++, lodLevel);
                 tasks.push(task);
             }
         }
@@ -564,15 +661,21 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     const newMeshInformation = getMeshInformation(newMesh);
     if (newMeshInformation.totalVertexCount < 4) {
         console.error(`[${NAME}] mesh LOD ${level} is has less than 4 vertices → skipping`);
-        return { level, path: fileName, success: false, cacheKey: cacheKey };
+        return { level, path: fileName, success: false, cacheKey: cacheKey, lodMeshInfo: null, density: null, };
+    }
+    let density = 0;
+    for (const prim of newMesh.listPrimitives()) {
+        const d = calculateMeshDensity(prim);
+        density = Math.max(density, d);
     }
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
-    if (!progressiveExt || !progressiveExt.create) {
+    if (!progressiveExt || !progressiveExt.createMesh) {
         console.error("Failed to create progressive extension");
         return;
     }
-    newMesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
+    newMesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createMesh({
         guid: guid,
+        density: density,
     }));
     // we need to compress the mesh with the same compression method as the original file
     const root = doc.getRoot();
@@ -585,17 +688,18 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     }
     const newIo = new NodeIO();
     registerExtensions(newIo);
-    console.log(`[${NAME}] Write mesh LOD ${level} with: ${JSON.stringify(lodLevel)} to \"${fileName}\"`);
+    console.log(`[${NAME}] Write mesh LOD ${level} with to \"${fileName}\"`);
     await newIo.write(outPath, newDoc);
     // const hasChanged = meshInfo.maxVertexCount == newMeshInformation.maxVertexCount && meshInfo.totalVertexCount == newMeshInformation.totalVertexCount;
     // add to cache
     if (cacheKey != undefined) {
         const bytes = readFileSync(outPath);
         addToCache(cacheKey, bytes);
+        addMetaToCache(cacheKey, { meshInfo: newMeshInformation, density: density });
     }
-    return { level, path: fileName, success: true, lodMeshInfo: newMeshInformation, cacheKey: cacheKey };
+    return { level, path: fileName, success: true, lodMeshInfo: newMeshInformation, cacheKey: cacheKey, density: density };
 }
-async function simplifyMesh(doc, mesh, prim, lodinfo) {
+async function simplifyMesh(doc, mesh, prim, index, lodinfo) {
     const weldOptions = {
         ...WELD_DEFAULTS,
         overwrite: true,
@@ -606,13 +710,7 @@ async function simplifyMesh(doc, mesh, prim, lodinfo) {
         ...lodinfo,
         simplifier: MeshoptSimplifier,
     };
-    // if (lodinfo.error != undefined)
-    // 	simplifyOptions.error = lodinfo.error;
-    // if (lodinfo.ratio != undefined)
-    // 	simplifyOptions.ratio = lodinfo.ratio;
-    // if (lodinfo.lockBorder != undefined)
-    // 	simplifyOptions.lockBorder = lodinfo.lockBorder;
-    return compressPrimitive(doc, mesh, prim, weldOptions, simplifyOptions);
+    return compressPrimitive(doc, mesh, prim, index, weldOptions, simplifyOptions);
 }
 export function isMeshLOD(filepath) {
     return filepath.includes("mesh_lod_");

package/dist/utils/mesh.d.ts

@@ -2,6 +2,7 @@ import { Mesh, Primitive } from "@gltf-transform/core";
 export declare type MeshInformation = {
     readonly maxVertexCount: number;
     readonly totalVertexCount: number;
+    readonly totalIndexCount: number;
 };
 export declare function getMeshInformation(mesh: Mesh): MeshInformation;
 export declare function calculateMeshDensity(prim: Primitive): number;

package/dist/utils/mesh.js

@@ -1,6 +1,9 @@
 export function getMeshInformation(mesh) {
     let maxVertexCount = 0;
+    let totalIndexCount = 0;
     const totalVertexCount = mesh.listPrimitives().reduce((acc, prim) => {
+        const indices = prim.getIndices()?.getCount() || 0;
+        totalIndexCount += indices;
         const vertices = prim.getAttribute('POSITION')?.getCount() || 0;
         maxVertexCount = Math.max(maxVertexCount, vertices);
         return acc + vertices;
@@ -8,6 +11,7 @@ export function getMeshInformation(mesh) {
     return {
         maxVertexCount,
         totalVertexCount,
+        totalIndexCount,
     };
 }
 export function calculateMeshDensity(prim) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.4.1-beta.9",
+  "version": "1.4.1",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",