@needle-tools/gltf-build-pipeline 1.4.1-beta.11 → 1.4.1-beta.12

package/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.4.1-beta.12] - 2023-03-23
+- add: NEEDLE_progressive LODs can now contain more info per texture or mesh. Meshes contain vertex- and index counts
+- change: the cache can now store and retrieve meta information
+
 ## [1.4.1-beta.11] - 2023-03-20
 - add: NEEDLE_progressive LOD meshes morph targets data
 - fix: NEEDLE_progressive should check if simplifcation did result in improved meshes (> 32 verts removed && > 5 % improvement) and otherwise discard LODs (and the whole extension if no improvement was made)

package/dist/cache/cache.d.ts

@@ -1,10 +1,47 @@
 import { ExtensibleProperty } from "@gltf-transform/core";
 /**
- * Get the available space on the cache directory in bytes.
+ * Clear the cache.
+ * This will remove all files in the cache directory.
  */
 export declare function clearCache(): void;
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
 export declare function ensureCacheSize(maxSizeMB: number): void;
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
 export declare function addToCache(key: string, bytes: Uint8Array): void;
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
 export declare function tryGetFromCache(key: string): Uint8Array | null;
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export declare function addMetaToCache(key: string, meta: object): void;
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export declare function tryGetMetaFromCache(key: string, ignoreVersion?: boolean): object | null;
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
 export declare function getKey(property: ExtensibleProperty): string;
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @returns The hash as a number.
+ */
 export declare function getHash(property: ExtensibleProperty | object): number;

package/dist/cache/cache.js

@@ -2,6 +2,7 @@ import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-tr
 import tmp from 'tmp';
 import fs from 'fs';
 import path from 'path';
+import { getVersion } from "../utils/version.js";
 // import * as CHECKDISCSPACE from 'check-disk-space'
 /**
  * Get the available space on the cache directory in bytes.
@@ -17,11 +18,20 @@ import path from 'path';
 //     }
 //     return 0;
 // }
+const buildPipelineVersion = getVersion(false);
+/**
+ * Clear the cache.
+ * This will remove all files in the cache directory.
+ */
 export function clearCache() {
     const dir = getCacheDirectory();
     fs.rmSync(dir, { recursive: true });
     console.log(`Cleared cache at ${dir}`);
 }
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
 export function ensureCacheSize(maxSizeMB) {
     const dir = getCacheDirectory();
     let totalSize = 0;
@@ -50,12 +60,23 @@ export function ensureCacheSize(maxSizeMB) {
     }
     console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
 }
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
 export function addToCache(key, bytes) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
     console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
     fs.writeFileSync(path, bytes);
 }
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
 export function tryGetFromCache(key) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
@@ -65,6 +86,50 @@ export function tryGetFromCache(key) {
     }
     return null;
 }
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export function addMetaToCache(key, meta) {
+    const dir = getCacheDirectory();
+    const metaPath = `${dir}/${key}.meta`;
+    meta['needle-build-pipeline:version'] = buildPipelineVersion;
+    fs.writeFileSync(metaPath, JSON.stringify(meta));
+}
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export function tryGetMetaFromCache(key, ignoreVersion = false) {
+    try {
+        const dir = getCacheDirectory();
+        const path = `${dir}/${key}.meta`;
+        if (fs.existsSync(path)) {
+            const bytes = fs.readFileSync(path);
+            if (bytes?.length > 0) {
+                const text = bytes.toString('utf-8');
+                if (text) {
+                    const obj = JSON.parse(text);
+                    if (ignoreVersion)
+                        return obj;
+                    if (obj['needle-build-pipeline:version'] === buildPipelineVersion)
+                        return obj;
+                }
+            }
+        }
+        return null;
+    }
+    catch (e) {
+        console.error(`Error reading cache meta: ${e.message}`);
+        return null;
+    }
+}
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
 export function getKey(property) {
     let prefix = "prop";
     if (property instanceof Texture) {
@@ -74,6 +139,11 @@ export function getKey(property) {
     const key = `${prefix}-${hash}`;
     return key;
 }
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @returns The hash as a number.
+ */
 export function getHash(property) {
     let hash = 0;
     if (property instanceof ExtensibleProperty) {

package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.d.ts

@@ -1,29 +1,39 @@
-import { Extension, ExtensionProperty, Graph, IProperty, Property, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
+import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare const NEEDLE_PROGRESSIVE_EXTENSION_NAME = "NEEDLE_progressive";
-export declare interface IProgressiveLoadingInfo {
+export type ProgressiveLoadingInfo = {
     /** guid of the asset the LODs belong to */
     guid: string;
-    /** paths to the LOD level files */
-    lods?: Array<{
-        path: string;
-        /** the density of the lod level */
-        density?: number;
+    /** lod infos and paths */
+    lods?: Array<LodInfo>;
+};
+type LodInfo = {
+    path: string;
+};
+export type ProgressiveTextureModel = ProgressiveLoadingInfo & {
+    lods?: Array<LodInfo & {}>;
+};
+export type ProgressiveMeshModel = ProgressiveLoadingInfo & {
+    density: number;
+    lods?: Array<LodInfo & {
+        density: number;
+        vertexCount: number;
+        indexCount: number;
     }>;
-    density?: number;
-}
+};
 export declare class NEEDLE_progressive extends Extension {
     static EXTENSION_NAME: string;
     extensionName: string;
-    static create(graph: Graph<Property<IProperty>>, data: IProgressiveLoadingInfo): NEEDLE_progressive_texture_property;
-    create(data: IProgressiveLoadingInfo): NEEDLE_progressive_texture_property;
+    createTexture(data: ProgressiveTextureModel): NEEDLE_progressive_property;
+    createMesh(data: ProgressiveMeshModel): NEEDLE_progressive_property;
     read(_context: ReaderContext): this;
     write(context: WriterContext): this;
 }
-export declare class NEEDLE_progressive_texture_property extends ExtensionProperty {
+export declare class NEEDLE_progressive_property extends ExtensionProperty {
     static EXTENSION_NAME: string;
     readonly extensionName: string;
     parentTypes: PropertyType[];
     propertyType: string;
     init(): void;
-    model: IProgressiveLoadingInfo;
+    model: ProgressiveLoadingInfo;
 }
+export {};

package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.js

@@ -4,13 +4,15 @@ export const NEEDLE_PROGRESSIVE_EXTENSION_NAME = "NEEDLE_progressive";
 export class NEEDLE_progressive extends Extension {
     static EXTENSION_NAME = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     extensionName = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
-    static create(graph, data) {
-        const ext = new NEEDLE_progressive_texture_property(graph);
+    createTexture(data) {
+        const ext = new NEEDLE_progressive_property(this.document.getGraph());
         ext.model = data;
         return ext;
     }
-    create(data) {
-        return NEEDLE_progressive.create(this.document.getGraph(), data);
+    createMesh(data) {
+        const ext = new NEEDLE_progressive_property(this.document.getGraph());
+        ext.model = data;
+        return ext;
     }
     read(_context) {
         // console.log("READ " + NEEDLE_PROGRESSIVE_TEXTURE_EXTENSION_NAME);
@@ -57,7 +59,7 @@ export class NEEDLE_progressive extends Extension {
         return this;
     }
 }
-export class NEEDLE_progressive_texture_property extends ExtensionProperty {
+export class NEEDLE_progressive_property extends ExtensionProperty {
     static EXTENSION_NAME = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     extensionName = NEEDLE_PROGRESSIVE_EXTENSION_NAME;
     parentTypes = [PropertyType.TEXTURE, PropertyType.MESH];

package/dist/transforms/needle_progressive.js

@@ -4,7 +4,7 @@ import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive } from "../extens
 import { createOpaqueExtension, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
 import path from 'path';
 import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
-import { addToCache, getHash, getKey, tryGetFromCache } from '../cache/index.js';
+import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
 import { foreachGLTF, generateGuid } from '../utils/index.js';
 import { needle_asset } from './needle_asset.js';
@@ -139,7 +139,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 continue;
             }
             const progressiveExt = doc.createExtension(NEEDLE_progressive);
-            if (!progressiveExt || !progressiveExt.create) {
+            if (!progressiveExt || !progressiveExt.createTexture) {
                 logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${textureIndex}`);
                 continue;
             }
@@ -150,7 +150,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             }
             const relName = path.relative(directory, outPath);
             // TODO: this path should be added to the NEEDLE dependencies extension
-            texture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
+            texture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createTexture({
                 guid: textureGuid,
                 lods: [
                     {
@@ -231,7 +231,7 @@ function createTextureLod(basePath, level, texture, slots, guid, textureIndex) {
     }
     // add progressive texture extension
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
-    newTexture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
+    newTexture.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createTexture({
         guid: guid,
     }));
     const newIo = new NodeIO();
@@ -355,7 +355,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
                 continue;
             }
             const progressiveExt = doc.createExtension(NEEDLE_progressive);
-            if (!progressiveExt || !progressiveExt.create) {
+            if (!progressiveExt || !progressiveExt.createMesh) {
                 logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${index++}`);
                 continue;
             }
@@ -479,12 +479,14 @@ export function make_progressive_meshes(filePath, _options, opts) {
                 continue;
             }
             if (generatedLODInfos.length > 0) {
-                const model = progressiveExt.create({
+                const model = progressiveExt.createMesh({
                     guid: settings.guid,
                     lods: generatedLODInfos.map(l => {
                         return {
                             path: l.path,
                             density: l.density,
+                            vertexCount: l.lodMeshInfo?.totalVertexCount,
+                            indexCount: l.lodMeshInfo?.totalIndexCount,
                         };
                     }),
                     density: maxDensity,
@@ -526,10 +528,11 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
         hash += getHash(lodLevel);
         cacheKey = `mesh_lod_${level}_${hash}`;
         const cachedValue = tryGetFromCache(cacheKey);
-        if (cachedValue) {
+        const meta = tryGetMetaFromCache(cacheKey);
+        if (cachedValue && meta) {
             console.log(`[${NAME}] LOD ${level} loaded ${(cachedValue.length / 1024).toFixed()} KB from cache for \"${mesh.getName()}\"`);
             writeFileSync(outPath, cachedValue);
-            return { level, path: fileName, success: true, cacheKey: cacheKey };
+            return { level, path: fileName, success: true, cacheKey: cacheKey, lodMeshInfo: meta.meshInfo, density: meta.density };
         }
     }
     const newDoc = new Document();
@@ -614,7 +617,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     const newMeshInformation = getMeshInformation(newMesh);
     if (newMeshInformation.totalVertexCount < 4) {
         console.error(`[${NAME}] mesh LOD ${level} is has less than 4 vertices → skipping`);
-        return { level, path: fileName, success: false, cacheKey: cacheKey };
+        return { level, path: fileName, success: false, cacheKey: cacheKey, lodMeshInfo: null, density: null, };
     }
     let density = 0;
     for (const prim of newMesh.listPrimitives()) {
@@ -622,11 +625,11 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
         density = Math.max(density, d);
     }
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
-    if (!progressiveExt || !progressiveExt.create) {
+    if (!progressiveExt || !progressiveExt.createMesh) {
         console.error("Failed to create progressive extension");
         return;
     }
-    newMesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.create({
+    newMesh.setExtension(NEEDLE_PROGRESSIVE_EXTENSION_NAME, progressiveExt.createMesh({
         guid: guid,
         density: density,
     }));
@@ -648,6 +651,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     if (cacheKey != undefined) {
         const bytes = readFileSync(outPath);
         addToCache(cacheKey, bytes);
+        addMetaToCache(cacheKey, { meshInfo: newMeshInformation, density: density });
     }
     return { level, path: fileName, success: true, lodMeshInfo: newMeshInformation, cacheKey: cacheKey, density: density };
 }

package/dist/utils/mesh.d.ts

@@ -2,6 +2,7 @@ import { Mesh, Primitive } from "@gltf-transform/core";
 export declare type MeshInformation = {
     readonly maxVertexCount: number;
     readonly totalVertexCount: number;
+    readonly totalIndexCount: number;
 };
 export declare function getMeshInformation(mesh: Mesh): MeshInformation;
 export declare function calculateMeshDensity(prim: Primitive): number;

package/dist/utils/mesh.js

@@ -1,6 +1,9 @@
 export function getMeshInformation(mesh) {
     let maxVertexCount = 0;
+    let totalIndexCount = 0;
     const totalVertexCount = mesh.listPrimitives().reduce((acc, prim) => {
+        const indices = prim.getIndices()?.getCount() || 0;
+        totalIndexCount += indices;
         const vertices = prim.getAttribute('POSITION')?.getCount() || 0;
         maxVertexCount = Math.max(maxVertexCount, vertices);
         return acc + vertices;
@@ -8,6 +11,7 @@ export function getMeshInformation(mesh) {
     return {
         maxVertexCount,
         totalVertexCount,
+        totalIndexCount,
     };
 }
 export function calculateMeshDensity(prim) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.4.1-beta.11",
+  "version": "1.4.1-beta.12",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",