@needle-tools/gltf-build-pipeline 2.8.0-alpha.1 → 2.8.1-alpha.0

package/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.8.1-alpha.0] - 2024-03-26
+- Fix: Issue where cache key gets longer depending on the primitive count
+- Fix: Cache key not taking index buffer into account
+- Fix: Caching of mesh LODs did clone shared mesh attributes for every primitive instead of re-using previous clones
+
+## [2.8.0-alpha.2] - 2024-03-04
+- change: use webp compression for textures smaller than 32x32
+- fix: webp compression for small textures (64x64)
+
 ## [2.8.0-alpha.0] - 2024-02-27
 - change: `product` usecase now uses webp compression for 'normalTexture' and 'metallicRoughnessTexture' textures
 

package/dist/cache/cache.js

@@ -202,8 +202,9 @@ export function getHash(property, level = 0) {
             const weights = property.getWeights();
             hash += hashObject(weights);
             const primitives = property.listPrimitives();
-            for (const prim of primitives)
-                hashStr += getHash(prim, level + 1);
+            for (const prim of primitives) {
+                hash += hashString(getHash(prim, level + 1));
+            }
         }
         else if (property instanceof Primitive) {
             const indices = property.getIndices();
@@ -222,11 +223,19 @@ export function getHash(property, level = 0) {
                 targets,
                 semantics: semanticsNames
             });
+            if (indices) {
+                const indexBuffer = indices.getArray();
+                if (indexBuffer) {
+                    hash += hashBuffer(indexBuffer.buffer, indexBuffer.byteOffset, indexBuffer.byteLength);
+                }
+            }
             // We can not pass the whole buffer into hashObject since very large meshes will cause errors when trying to stringify
             // See https://linear.app/needle/issue/NE-6414
             for (const sem of semantics) {
-                const arr = property.getAttribute(sem).getArray();
-                hash += hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength);
+                const arr = property.getAttribute(sem)?.getArray();
+                if (arr) {
+                    hash += hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength);
+                }
             }
         }
     }

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -954,7 +954,7 @@ class MeshIndexResolver extends PointerResolver {
                 if (parent instanceof Node) {
                     obj = parent.getMesh();
                     if (obj) {
-                        console.log(`INFO: resolved mesh \"${this.inputObject.getName()}\" from parent node \"${parent.getName()}\"`);
+                        this.document.getLogger().info(`INFO: resolved mesh \"${this.inputObject.getName()}\" from parent node \"${parent.getName()}\"`);
                         break;
                     }
                 }

package/dist/transforms/needle_progressive.js

@@ -850,27 +850,14 @@ async function createMeshLod(inputFileName, doc, basePath, level, mesh, meshInfo
     }
     const tasks = new Array();
     // let didRunSimplification = false;
+    const clonedAttributes = new Map();
+    debugger;
     for (const prim of primitives) {
         const newPrim = newDoc.createPrimitive();
         newMesh.addPrimitive(newPrim);
         newPrim.setName(prim.getName());
         newPrim.setMode(prim.getMode());
         newPrim.setMaterial(newMat);
-        const semantics = prim.listSemantics();
-        for (let i = 0; i < semantics.length; i++) {
-            const attr = prim.getAttribute(semantics[i]);
-            if (attr) {
-                const newAttr = newDoc.createAccessor();
-                newAttr.setArray(attr.getArray());
-                newAttr.setType(attr.getType());
-                newAttr.setSparse(attr.getSparse());
-                newAttr.setNormalized(attr.getNormalized());
-                newPrim.setAttribute(semantics[i], newAttr);
-            }
-            else {
-                logger.warn(`[${NAME}] Missing attribute ${semantics[i]} in mesh LOD ${level}`);
-            }
-        }
         const indices = prim.getIndices();
         if (indices) {
             const newIndices = newDoc.createAccessor();
@@ -880,6 +867,28 @@ async function createMeshLod(inputFileName, doc, basePath, level, mesh, meshInfo
             newIndices.setNormalized(indices.getNormalized());
             newPrim.setIndices(newIndices);
         }
+        const semantics = prim.listSemantics();
+        for (let i = 0; i < semantics.length; i++) {
+            const attr = prim.getAttribute(semantics[i]);
+            if (attr) {
+                if (clonedAttributes.has(attr)) {
+                    const cloned = clonedAttributes.get(attr);
+                    newPrim.setAttribute(semantics[i], cloned);
+                }
+                else {
+                    const newAttr = newDoc.createAccessor();
+                    newAttr.setArray(attr.getArray());
+                    newAttr.setType(attr.getType());
+                    newAttr.setSparse(attr.getSparse());
+                    newAttr.setNormalized(attr.getNormalized());
+                    newPrim.setAttribute(semantics[i], newAttr);
+                    clonedAttributes.set(attr, newAttr);
+                }
+            }
+            else {
+                logger.warn(`[${NAME}] Missing attribute ${semantics[i]} in mesh LOD ${level}`);
+            }
+        }
         const targets = prim.listTargets();
         for (const target of targets) {
             const newTarget = newDoc.createPrimitiveTarget(target.getName());
@@ -964,7 +973,8 @@ async function createMeshLod(inputFileName, doc, basePath, level, mesh, meshInfo
     // add to cache
     if (cacheKey != undefined) {
         const bytes = readFileSync(outPath);
-        logger.info(`[${NAME}] Add LOD ${level} to cache for \"${mesh.getName()}\" as ${cacheKey}`);
+        const sizeInMB = (bytes.byteLength / 1024 / 1024).toFixed(2);
+        logger.info(`[${NAME}] Add LOD ${level} to cache (${sizeInMB} MB) for \"${mesh.getName()}\" as ${cacheKey}`);
         addToCache(cacheKey, bytes);
         addMetaToCache(cacheKey, { meshInfo: newMeshInformation, density: density });
     }

package/dist/transforms/needle_texture_transform.js

@@ -147,6 +147,7 @@ async function processTexture(texture, index, document, logger, context) {
     else {
         let mimeType = "image/ktx2";
         let ktx2Format = undefined;
+        let webpOptions = {};
         const slots = listTextureSlots(texture);
         const size = texture.getSize();
         const isVRM = context.file?.toLowerCase().endsWith(".vrm");
@@ -166,13 +167,8 @@ async function processTexture(texture, index, document, logger, context) {
         }
         // for https://linear.app/needle/issue/NE-6464
         else if (size[0] <= 32 || size[1] <= 32) {
-            if (canRunToktx) {
-                mimeType = "image/ktx2";
-                ktx2Format = "UASTC";
-            }
-            else {
-                mimeType = "image/webp";
-            }
+            // can not use UASTC because it's ALSO shitty, see issue above
+            mimeType = "image/webp";
         }
         // For product compression we automatically select webP if the texture is used in a PBR material
         // https://linear.app/needle/issue/NE-6458
@@ -191,6 +187,7 @@ async function processTexture(texture, index, document, logger, context) {
         await resizeTextureIfNecessary(texture);
         if (mimeType === "image/webp") {
             successFullyProcessed = await context.webPContext.process(index, texture, {
+                ...webpOptions,
                 quality: .9,
             }, context);
         }

package/dist/transforms/needle_webp.d.ts

@@ -1,6 +1,7 @@
 import { Document, ILogger, Texture } from '@gltf-transform/core';
 import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
 import { INeedleTextureTransformContext, NeedleTransformContext } from './needle_common.js';
+import sharp from 'sharp';
 export interface NeedleWebPOptions {
 }
 export declare class NeedleWebPContext implements INeedleTextureTransformContext {
@@ -9,6 +10,6 @@ export declare class NeedleWebPContext implements INeedleTextureTransformContext
     private compressed;
     constructor(options: NeedleWebPOptions);
     prepare(document: Document): Promise<void>;
-    process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality">, context: NeedleTransformContext): Promise<boolean>;
+    process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality"> & Partial<sharp.WebpOptions>, context: NeedleTransformContext): Promise<boolean>;
     finalize(document: Document): Promise<void>;
 }

package/dist/transforms/needle_webp.js

@@ -13,7 +13,15 @@ export class NeedleWebPContext {
         this.logger = document.getLogger();
     }
     async process(index, texture, settings, context) {
+        const size = texture.getSize();
+        // if the texture is too small we will use lossless compression
+        let lossless = false;
+        if (size[0] < 64 || size[1] < 64) {
+            lossless = true;
+        }
         const options = {
+            lossless,
+            ...settings,
             quality: settings.quality,
             // effort: remap(options.effort, 100, 6),
             // lossless: options.lossless,

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.8.0-alpha.1";
+export declare const version = "2.8.1-alpha.0";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.8.0-alpha.1";
+export const version = "2.8.1-alpha.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.8.0-alpha.1",
+  "version": "2.8.1-alpha.0",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",