@needle-tools/gltf-build-pipeline 2.2.0-alpha.1 → 2.2.0-alpha.11

package/CHANGELOG.md

@@ -4,12 +4,25 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [2.2.0-alpha] - 2023-06-06
-- Add: CLI `--progressive` option to `transform` command (default is enabled)
-- Add: config `meshes: { lods: boolean }` option
-- Remove: CLI "compress" command
-- Change: enable texture LODs by default
-- Fix: `--config` path can now point to a directory or a `needle.config.json` file as well
+## [2.2.0-alpha.11] - 2023-06-14
+- fix: issue where progressive transform with un-indexed meshes would fail 
+
+## [2.2.0-alpha.10] - 2023-06-13
+- fix: progressive transform of converted GLB files where meshes don't have indicies
+- fix: skip progressive texture generation for textures for textures that already match the lowest LOD size
+- fix: if compression is set to None the LOD textures should respect that setting and also not be compressed
+- fix: fallback mesh density calculation for meshes that aren't indexed
+- fix: NEEDLE_mesh_compression extension falsely overriding primitive extensions
+
+## [2.2.0-alpha.5] - 2023-06-06
+- add: CLI `--progressive` option to `transform` command (default is true)
+- add: CLI `--compress` option to `transform` command (default is true)
+- add: config `meshes: { lods: boolean }` option
+- remove: CLI "compress" command
+- change: enable texture LODs by default
+- fix: `--config` path can now point to a directory or a `needle.config.json` file as well
+- fix: mesh LOD regression caused by change in gltf-transform `weld` behaviour 
+- fix: KTX SOFTWARE >= 4.3.0 is installed
 
 ## [2.1.0-alpha.6] - 2023-06-04
 - fix: errors when invoking npm scripts caused by the cli program

package/dist/cache/cache.js

@@ -19,6 +19,7 @@ import { getVersion } from "../utils/version.js";
 //     return 0;
 // }
 const buildPipelineVersion = getVersion(false);
+const debugLogs = process.argv.includes("--debug");
 /**
  * Clear the cache.
  * This will remove all files in the cache directory.
@@ -86,7 +87,8 @@ export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
 export function addToCache(key, bytes) {
     const dir = getCacheDirectory();
     const path = `${dir}/${key}.bin`;
-    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
+    if (debugLogs)
+        console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
     fs.writeFileSync(path, bytes);
 }
 /**

package/dist/cli/index.js

@@ -32,6 +32,7 @@ Each version will be compressed and written to the output directory.
 `)
     .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
     // .argument('<output>', "The output file. If not provided, the input file will be overwritten.", { default: "" })
+    .option("--compress", `When enabled files will be compressed`, { validator: program.BOOLEAN, default: true })
     .option("--progressive", `When enabled files will processed to be progressively loaded`, { validator: program.BOOLEAN, default: true })
     .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
@@ -39,6 +40,10 @@ Each version will be compressed and written to the output directory.
     .option("--cache", "Use \"--cache False\" to disable caching.\nWhen caching is enabled then previously processed files will be detected and loaded from disc cache instead of being processed again if settings or the file contents have not changed.", { validator: program.BOOLEAN, default: true })
     .option("--debug", "Enable for more verbose outout with \"--debug True\"", { validator: program.BOOLEAN, default: false })
     .action(async ({ logger, args, options }) => {
+    if (options.compress === false && options.progressive === false) {
+        logger.error("You need to enable at least one of the options: --compress or --progressive");
+        return;
+    }
     logger.info("Transforming glTF file: " + args.input);
     const stats = {
         startTime: Date.now(),
@@ -72,7 +77,7 @@ Each version will be compressed and written to the output directory.
         logger.info("→ Transform " + file);
         const progressive_results = new Array();
         if (options.progressive === true) {
-            await make_progressive({
+            if (await make_progressive({
                 path: file,
                 results: progressive_results,
                 config,
@@ -81,25 +86,29 @@ Each version will be compressed and written to the output directory.
             }).catch(err => {
                 logger.error(err);
                 process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
-            });
+            }) === false) {
+                process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
+            }
         }
-        const opts = {
-            config,
-            stats,
-            packedFiles: set,
-            debug,
-            useCache,
-            logger: logger,
-            file_stats,
-        };
-        for (const res of progressive_results) {
-            if (await packGLTF(res, res, opts) === false) {
+        if (options.compress === true) {
+            const opts = {
+                config,
+                stats,
+                packedFiles: set,
+                debug,
+                useCache,
+                logger: logger,
+                file_stats,
+            };
+            for (const res of progressive_results) {
+                if (await packGLTF(res, res, opts) === false) {
+                    process.exit(ERROR_CODES.PACKING_FAILED);
+                }
+            }
+            if (await packGLTF(file, file, opts) === false) {
                 process.exit(ERROR_CODES.PACKING_FAILED);
             }
         }
-        if (await packGLTF(file, file, opts) === false) {
-            process.exit(ERROR_CODES.PACKING_FAILED);
-        }
     });
     printStats(stats, logger, config);
     if (options.stats) {

package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js

@@ -62,7 +62,7 @@ export class NEEDLE_mesh_compression extends Extension {
                 if (ext) {
                     const prim = meshDef.primitives[i];
                     if (prim) {
-                        prim.extensions = meshDef.extensions || {};
+                        prim.extensions = prim.extensions || {};
                         prim.extensions[EXTENSION_NAME] = ext.model;
                     }
                 }

package/dist/functions/weld.3.d.ts

@@ -0,0 +1,76 @@
+import { Document, Primitive, Transform } from '@gltf-transform/core';
+/** @hidden */
+export declare function createIndices(count: number, maxIndex?: number): Uint16Array | Uint32Array;
+export declare function formatLong(x: number): string;
+export declare function formatDelta(a: number, b: number, decimals?: number): string;
+export declare function formatDeltaOp(a: number, b: number): string;
+export declare function cleanPrimitive(prim: Primitive): void;
+/** Options for the {@link weld} function. */
+export interface WeldOptions {
+    /** Tolerance for vertex positions, as a fraction of primitive AABB. */
+    tolerance?: number;
+    /** Tolerance for vertex normals, in radians. */
+    toleranceNormal?: number;
+    /** Whether to overwrite existing indices. */
+    overwrite?: boolean;
+    /** Enables a more thorough, but slower, search for vertices to weld. */
+    exhaustive?: boolean;
+}
+export declare const WELD_DEFAULTS: Required<WeldOptions>;
+/**
+ * Index {@link Primitive Primitives} and (optionally) merge similar vertices. When merged
+ * and indexed, data is shared more efficiently between vertices. File size can
+ * be reduced, and the GPU can sometimes use the vertex cache more efficiently.
+ *
+ * When welding, the 'tolerance' threshold determines which vertices qualify for
+ * welding based on distance between the vertices as a fraction of the primitive's
+ * bounding box (AABB). For example, tolerance=0.01 welds vertices within +/-1%
+ * of the AABB's longest dimension. Other vertex attributes are also compared
+ * during welding, with attribute-specific thresholds. For `tolerance=0`, geometry
+ * is indexed in place, without merging.
+ *
+ * To preserve visual appearance consistently, use low `toleranceNormal` thresholds
+ * around 0.1 (±3º). To pre-processing a scene before simplification or LOD creation,
+ * use higher thresholds around 0.5 (±30º).
+ *
+ * Example:
+ *
+ * ```javascript
+ * import { weld } from '@gltf-transform/functions';
+ *
+ * await document.transform(
+ * 	weld({ tolerance: 0.001, toleranceNormal: 0.5 })
+ * );
+ * ```
+ *
+ * @category Transforms
+ */
+export declare function weld(_options?: WeldOptions): Transform;
+/**
+ * Index a {@link Primitive} and (optionally) weld similar vertices. When merged
+ * and indexed, data is shared more efficiently between vertices. File size can
+ * be reduced, and the GPU can sometimes use the vertex cache more efficiently.
+ *
+ * When welding, the 'tolerance' threshold determines which vertices qualify for
+ * welding based on distance between the vertices as a fraction of the primitive's
+ * bounding box (AABB). For example, tolerance=0.01 welds vertices within +/-1%
+ * of the AABB's longest dimension. Other vertex attributes are also compared
+ * during welding, with attribute-specific thresholds. For tolerance=0, geometry
+ * is indexed in place, without merging.
+ *
+ * Example:
+ *
+ * ```javascript
+ * import { weldPrimitive } from '@gltf-transform/functions';
+ *
+ * const mesh = document.getRoot().listMeshes()
+ * 	.find((mesh) => mesh.getName() === 'Gizmo');
+ *
+ * for (const prim of mesh.listPrimitives()) {
+ *   weldPrimitive(prim, {tolerance: 0.0001});
+ * }
+ * ```
+ *
+ * @privateRemarks TODO(v4): Remove the "Document" parameter.
+ */
+export declare function weldPrimitive(a: Document | Primitive, b?: Primitive | WeldOptions, c?: Required<WeldOptions>): void;

package/dist/functions/weld.3.js

@@ -0,0 +1,414 @@
+/*
+
+
+COPY OF weld() of gltf-transform 3.10.1
+2aec33bf432a354275fb5066d1ddf5840f2d8bb4
+
+
+
+*/
+import { Accessor, Document, Primitive, PropertyType, } from '@gltf-transform/core';
+import { createTransform, prune, dedup } from '@gltf-transform/functions';
+/** @hidden */
+export function createIndices(count, maxIndex = count) {
+    const array = maxIndex <= 65534 ? new Uint16Array(count) : new Uint32Array(count);
+    for (let i = 0; i < array.length; i++)
+        array[i] = i;
+    return array;
+}
+export function formatLong(x) {
+    return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
+}
+export function formatDelta(a, b, decimals = 2) {
+    const prefix = a > b ? '–' : '+';
+    const suffix = '%';
+    return prefix + ((Math.abs(a - b) / a) * 100).toFixed(decimals) + suffix;
+}
+export function formatDeltaOp(a, b) {
+    return `${formatLong(a)} → ${formatLong(b)} (${formatDelta(a, b)})`;
+}
+export function cleanPrimitive(prim) {
+    const indices = prim.getIndices();
+    if (!indices)
+        return;
+    const tmpIndicesArray = [];
+    let maxIndex = -Infinity;
+    for (let i = 0, il = indices.getCount(); i < il; i += 3) {
+        const a = indices.getScalar(i);
+        const b = indices.getScalar(i + 1);
+        const c = indices.getScalar(i + 2);
+        if (a === b || a === c || b === c)
+            continue;
+        // @ts-ignore
+        tmpIndicesArray.push(a, b, c);
+        maxIndex = Math.max(maxIndex, a, b, c);
+    }
+    const dstIndicesArray = createIndices(tmpIndicesArray.length, maxIndex);
+    dstIndicesArray.set(tmpIndicesArray);
+    indices.setArray(dstIndicesArray);
+}
+// DEVELOPER NOTES: Ideally a weld() implementation should be fast, robust,
+// and tunable. The writeup below tracks my attempts to solve for these
+// constraints.
+//
+// (Approach #1) Follow the mergeVertices() implementation of three.js,
+// hashing vertices with a string concatenation of all vertex attributes.
+// The approach does not allow per-attribute tolerance in local units.
+//
+// (Approach #2) Sort points along the X axis, then make cheaper
+// searches up/down the sorted list for merge candidates. While this allows
+// simpler comparison based on specified tolerance, it's much slower, even
+// for cases where choice of the X vs. Y or Z axes is reasonable.
+//
+// (Approach #3) Attempted a Delaunay triangulation in three dimensions,
+// expecting it would be an n * log(n) algorithm, but the only implementation
+// I found (with delaunay-triangulate) appeared to be much slower than that,
+// and was notably slower than the sort-based approach, just building the
+// Delaunay triangulation alone.
+//
+// (Approach #4) Hybrid of (1) and (2), assigning vertices to a spatial
+// grid, then searching the local neighborhood (27 cells) for weld candidates.
+//
+// RESULTS: For the "Lovecraftian" sample model, after joining, a primitive
+// with 873,000 vertices can be welded down to 230,000 vertices. Results:
+// - (1) Not tested, but prior results suggest not robust enough.
+// - (2) 30 seconds
+// - (3) 660 seconds
+// - (4) 5 seconds exhaustive, 1.5s non-exhaustive
+const NAME = 'weld';
+const Tolerance = {
+    DEFAULT: 0.0001,
+    TEXCOORD: 0.0001,
+    COLOR: 0.01,
+    NORMAL: 0.05,
+    JOINTS: 0.0,
+    WEIGHTS: 0.01, // [0, ∞]
+};
+export const WELD_DEFAULTS = {
+    tolerance: Tolerance.DEFAULT,
+    toleranceNormal: Tolerance.NORMAL,
+    overwrite: true,
+    exhaustive: false, // donmccurdy/glTF-Transform#886
+};
+/**
+ * Index {@link Primitive Primitives} and (optionally) merge similar vertices. When merged
+ * and indexed, data is shared more efficiently between vertices. File size can
+ * be reduced, and the GPU can sometimes use the vertex cache more efficiently.
+ *
+ * When welding, the 'tolerance' threshold determines which vertices qualify for
+ * welding based on distance between the vertices as a fraction of the primitive's
+ * bounding box (AABB). For example, tolerance=0.01 welds vertices within +/-1%
+ * of the AABB's longest dimension. Other vertex attributes are also compared
+ * during welding, with attribute-specific thresholds. For `tolerance=0`, geometry
+ * is indexed in place, without merging.
+ *
+ * To preserve visual appearance consistently, use low `toleranceNormal` thresholds
+ * around 0.1 (±3º). To pre-processing a scene before simplification or LOD creation,
+ * use higher thresholds around 0.5 (±30º).
+ *
+ * Example:
+ *
+ * ```javascript
+ * import { weld } from '@gltf-transform/functions';
+ *
+ * await document.transform(
+ * 	weld({ tolerance: 0.001, toleranceNormal: 0.5 })
+ * );
+ * ```
+ *
+ * @category Transforms
+ */
+export function weld(_options = WELD_DEFAULTS) {
+    const options = expandWeldOptions(_options);
+    return createTransform(NAME, async (doc) => {
+        const logger = doc.getLogger();
+        for (const mesh of doc.getRoot().listMeshes()) {
+            for (const prim of mesh.listPrimitives()) {
+                weldPrimitive(doc, prim, options);
+                if (isPrimEmpty(prim))
+                    prim.dispose();
+            }
+            if (mesh.listPrimitives().length === 0)
+                mesh.dispose();
+        }
+        if (options.tolerance > 0) {
+            // If tolerance is greater than 0, welding may remove a mesh, so we prune
+            await doc.transform(prune({
+                propertyTypes: [PropertyType.ACCESSOR, PropertyType.NODE],
+                keepAttributes: true,
+                keepIndices: true,
+                keepLeaves: false,
+            }));
+        }
+        await doc.transform(dedup({ propertyTypes: [PropertyType.ACCESSOR] }));
+        logger.debug(`${NAME}: Complete.`);
+    });
+}
+/**
+ * Index a {@link Primitive} and (optionally) weld similar vertices. When merged
+ * and indexed, data is shared more efficiently between vertices. File size can
+ * be reduced, and the GPU can sometimes use the vertex cache more efficiently.
+ *
+ * When welding, the 'tolerance' threshold determines which vertices qualify for
+ * welding based on distance between the vertices as a fraction of the primitive's
+ * bounding box (AABB). For example, tolerance=0.01 welds vertices within +/-1%
+ * of the AABB's longest dimension. Other vertex attributes are also compared
+ * during welding, with attribute-specific thresholds. For tolerance=0, geometry
+ * is indexed in place, without merging.
+ *
+ * Example:
+ *
+ * ```javascript
+ * import { weldPrimitive } from '@gltf-transform/functions';
+ *
+ * const mesh = document.getRoot().listMeshes()
+ * 	.find((mesh) => mesh.getName() === 'Gizmo');
+ *
+ * for (const prim of mesh.listPrimitives()) {
+ *   weldPrimitive(prim, {tolerance: 0.0001});
+ * }
+ * ```
+ *
+ * @privateRemarks TODO(v4): Remove the "Document" parameter.
+ */
+export function weldPrimitive(a, b = WELD_DEFAULTS, c = WELD_DEFAULTS) {
+    let _document;
+    let _prim;
+    let _options;
+    if (a instanceof Primitive) {
+        const graph = a.getGraph();
+        _document = Document.fromGraph(graph);
+        _prim = a;
+        _options = expandWeldOptions(b);
+    }
+    else {
+        _document = a;
+        _prim = b;
+        _options = expandWeldOptions(c);
+    }
+    if (_prim.getIndices() && !_options.overwrite)
+        return;
+    if (_prim.getMode() === Primitive.Mode.POINTS)
+        return;
+    if (_options.tolerance === 0) {
+        _indexPrimitive(_document, _prim);
+    }
+    else {
+        _weldPrimitive(_document, _prim, _options);
+    }
+}
+/** @internal Adds indices, if missing. Does not merge vertices. */
+function _indexPrimitive(doc, prim) {
+    // No need to overwrite here, even if options.overwrite=true.
+    if (prim.getIndices())
+        return;
+    const attr = prim.listAttributes()[0];
+    const numVertices = attr.getCount();
+    const buffer = attr.getBuffer();
+    const indices = doc
+        .createAccessor()
+        .setBuffer(buffer)
+        .setType(Accessor.Type.SCALAR)
+        .setArray(createIndices(numVertices));
+    prim.setIndices(indices);
+}
+/** @internal Weld and merge, combining vertices that are similar on all vertex attributes. */
+function _weldPrimitive(doc, prim, options) {
+    const logger = doc.getLogger();
+    const srcPosition = prim.getAttribute('POSITION');
+    const srcIndices = prim.getIndices() || doc.createAccessor().setArray(createIndices(srcPosition.getCount()));
+    const uniqueIndices = new Uint32Array(new Set(srcIndices.getArray())).sort();
+    // (1) Compute per-attribute tolerance and spatial grid for vertices.
+    const attributeTolerance = {};
+    for (const semantic of prim.listSemantics()) {
+        const attribute = prim.getAttribute(semantic);
+        attributeTolerance[semantic] = getAttributeTolerance(semantic, attribute, options);
+    }
+    logger.debug(`${NAME}: Tolerance thresholds: ${formatKV(attributeTolerance)}`);
+    // (2) Compare and identify vertices to weld.
+    const posA = [0, 0, 0];
+    const posB = [0, 0, 0];
+    const grid = {};
+    const cellSize = attributeTolerance.POSITION;
+    for (let i = 0; i < uniqueIndices.length; i++) {
+        srcPosition.getElement(uniqueIndices[i], posA);
+        const key = getGridKey(posA, cellSize);
+        grid[key] = grid[key] || [];
+        grid[key].push(uniqueIndices[i]);
+    }
+    // (2) Compare and identify vertices to weld.
+    const srcMaxIndex = uniqueIndices[uniqueIndices.length - 1];
+    const weldMap = createIndices(srcMaxIndex + 1); // oldIndex → oldCommonIndex
+    const writeMap = new Array(uniqueIndices.length).fill(-1); // oldIndex → newIndex
+    const srcVertexCount = srcPosition.getCount();
+    let dstVertexCount = 0;
+    for (let i = 0; i < uniqueIndices.length; i++) {
+        const a = uniqueIndices[i];
+        srcPosition.getElement(a, posA);
+        const cellKeys = options.exhaustive ? getGridNeighborhoodKeys(posA, cellSize) : [getGridKey(posA, cellSize)];
+        cells: for (const cellKey of cellKeys) {
+            if (!grid[cellKey])
+                continue cells; // May occur in exhaustive search.
+            neighbors: for (const j of grid[cellKey]) {
+                const b = weldMap[j];
+                // Only weld to lower indices, preventing two-way match.
+                if (a <= b)
+                    continue neighbors;
+                srcPosition.getElement(b, posB);
+                // Weld if base attributes and morph target attributes match.
+                const isBaseMatch = prim.listSemantics().every((semantic) => {
+                    const attribute = prim.getAttribute(semantic);
+                    const tolerance = attributeTolerance[semantic];
+                    return compareAttributes(attribute, a, b, tolerance, semantic);
+                });
+                const isTargetMatch = prim.listTargets().every((target) => {
+                    return target.listSemantics().every((semantic) => {
+                        const attribute = target.getAttribute(semantic);
+                        const tolerance = attributeTolerance[semantic];
+                        return compareAttributes(attribute, a, b, tolerance, semantic);
+                    });
+                });
+                if (isBaseMatch && isTargetMatch) {
+                    weldMap[a] = b;
+                    break cells;
+                }
+            }
+        }
+        // Output the vertex if we didn't find a match, else record the index of the match. Because
+        // we iterate vertices in ascending order, and only match to lower indices, we're
+        // guaranteed the source vertex for a weld has already been marked for output.
+        if (weldMap[a] === a) {
+            writeMap[a] = dstVertexCount++;
+        }
+        else {
+            writeMap[a] = writeMap[weldMap[a]];
+        }
+    }
+    logger.debug(`${NAME}: ${formatDeltaOp(srcVertexCount, dstVertexCount)} vertices.`);
+    // (3) Update indices.
+    const dstIndicesCount = srcIndices.getCount(); // # primitives does not change.
+    const dstIndicesArray = createIndices(dstIndicesCount, uniqueIndices.length);
+    for (let i = 0; i < dstIndicesCount; i++) {
+        dstIndicesArray[i] = writeMap[srcIndices.getScalar(i)];
+    }
+    prim.setIndices(srcIndices.clone().setArray(dstIndicesArray));
+    if (srcIndices.listParents().length === 1)
+        srcIndices.dispose();
+    // (4) Update vertex attributes.
+    for (const srcAttr of prim.listAttributes()) {
+        swapAttributes(prim, srcAttr, writeMap, dstVertexCount);
+    }
+    for (const target of prim.listTargets()) {
+        for (const srcAttr of target.listAttributes()) {
+            swapAttributes(target, srcAttr, writeMap, dstVertexCount);
+        }
+    }
+    // (5) Clean up degenerate triangles.
+    cleanPrimitive(prim);
+}
+/** Creates a new TypedArray of the same type as an original, with a new length. */
+function createArrayOfType(array, length) {
+    const ArrayCtor = array.constructor;
+    return new ArrayCtor(length);
+}
+/** Replaces an {@link Attribute}, creating a new one with the given elements. */
+function swapAttributes(parent, srcAttr, reorder, dstCount) {
+    const dstAttrArray = createArrayOfType(srcAttr.getArray(), dstCount * srcAttr.getElementSize());
+    const dstAttr = srcAttr.clone().setArray(dstAttrArray);
+    const done = new Uint8Array(dstCount);
+    for (let i = 0, el = []; i < reorder.length; i++) {
+        if (!done[reorder[i]]) {
+            dstAttr.setElement(reorder[i], srcAttr.getElement(i, el));
+            done[reorder[i]] = 1;
+        }
+    }
+    parent.swap(srcAttr, dstAttr);
+    // Clean up.
+    if (srcAttr.listParents().length === 1)
+        srcAttr.dispose();
+}
+const _a = [];
+const _b = [];
+/** Computes a per-attribute tolerance, based on domain and usage of the attribute. */
+function getAttributeTolerance(semantic, attribute, options) {
+    // Attributes like NORMAL and COLOR_# do not vary in range like POSITION,
+    // so do not apply the given tolerance factor to these attributes.
+    if (semantic === 'NORMAL' || semantic === 'TANGENT')
+        return options.toleranceNormal;
+    if (semantic.startsWith('COLOR_'))
+        return Tolerance.COLOR;
+    if (semantic.startsWith('TEXCOORD_'))
+        return Tolerance.TEXCOORD;
+    if (semantic.startsWith('JOINTS_'))
+        return Tolerance.JOINTS;
+    if (semantic.startsWith('WEIGHTS_'))
+        return Tolerance.WEIGHTS;
+    _a.length = _b.length = 0;
+    attribute.getMinNormalized(_a);
+    attribute.getMaxNormalized(_b);
+    const diff = _b.map((bi, i) => bi - _a[i]);
+    const range = Math.max(...diff);
+    return options.tolerance * range;
+}
+/** Compares two vertex attributes against a tolerance threshold. */
+function compareAttributes(attribute, a, b, tolerance, _semantic) {
+    attribute.getElement(a, _a);
+    attribute.getElement(b, _b);
+    for (let i = 0, il = attribute.getElementSize(); i < il; i++) {
+        if (Math.abs(_a[i] - _b[i]) > tolerance) {
+            return false;
+        }
+    }
+    return true;
+}
+function formatKV(kv) {
+    return Object.entries(kv)
+        .map(([k, v]) => `${k}=${v}`)
+        .join(', ');
+}
+// Order to search nearer cells first.
+const CELL_OFFSETS = [0, -1, 1];
+function getGridNeighborhoodKeys(p, cellSize) {
+    const keys = [];
+    const _p = [0, 0, 0];
+    for (const i of CELL_OFFSETS) {
+        for (const j of CELL_OFFSETS) {
+            for (const k of CELL_OFFSETS) {
+                _p[0] = p[0] + i * cellSize;
+                _p[1] = p[1] + j * cellSize;
+                _p[2] = p[2] + k * cellSize;
+                keys.push(getGridKey(_p, cellSize));
+            }
+        }
+    }
+    return keys;
+}
+function getGridKey(p, cellSize) {
+    const cellX = Math.round(p[0] / cellSize);
+    const cellY = Math.round(p[1] / cellSize);
+    const cellZ = Math.round(p[2] / cellSize);
+    return cellX + ':' + cellY + ':' + cellZ;
+}
+function expandWeldOptions(_options) {
+    const options = { ...WELD_DEFAULTS, ..._options };
+    if (options.tolerance < 0 || options.tolerance > 0.1) {
+        throw new Error(`${NAME}: Requires 0 <= tolerance <= 0.1`);
+    }
+    if (options.toleranceNormal < 0 || options.toleranceNormal > Math.PI / 2) {
+        throw new Error(`${NAME}: Requires 0 <= toleranceNormal <= ${(Math.PI / 2).toFixed(2)}`);
+    }
+    if (options.tolerance > 0) {
+        options.tolerance = Math.max(options.tolerance, Number.EPSILON);
+        options.toleranceNormal = Math.max(options.toleranceNormal, Number.EPSILON);
+    }
+    return options;
+}
+/**
+ * For purposes of welding, we consider a primitive to be 'empty' or degenerate
+ * if (1) it has an index, and (2) that index is empty. In some cases
+ * (mode=POINTS) the index may be missing — this is outside the scope of welding.
+ */
+function isPrimEmpty(prim) {
+    const indices = prim.getIndices();
+    return !!indices && indices.getCount() === 0;
+}

package/dist/transforms/needle_mesh_transform.js

@@ -1,7 +1,8 @@
 import { Primitive } from "@gltf-transform/core";
-import { meshopt, simplifyPrimitive, SIMPLIFY_DEFAULTS, weldPrimitive, WELD_DEFAULTS, draco } from "@gltf-transform/functions";
+import { meshopt, simplifyPrimitive, SIMPLIFY_DEFAULTS, WELD_DEFAULTS, draco } from "@gltf-transform/functions";
 import { MeshoptEncoder, MeshoptSimplifier } from "meshoptimizer";
 import { NEEDLE_mesh_compression } from "../extensions/index.js";
+import { weldPrimitive } from "../functions/weld.3.js";
 const NAME = 'needle_mesh_transform';
 export const needle_mesh_transform = function (context, simplify) {
     return async (document, _context) => {
@@ -52,6 +53,7 @@ async function simplifierTransform(document, _context, _options) {
         options.simplifier = simplifier;
     const weldOptions = {
         ...WELD_DEFAULTS,
+        tolerance: 0.00000000001,
         overwrite: true,
     };
     for (const mesh of document.getRoot().listMeshes()) {

package/dist/transforms/needle_progressive.d.ts

@@ -7,7 +7,7 @@ declare interface ProgressiveOptions {
     debug?: boolean;
     useCache?: boolean;
 }
-export declare function make_progressive(opts: ProgressiveOptions): Promise<void>;
+export declare function make_progressive(opts: ProgressiveOptions): Promise<boolean>;
 export declare function createTransform(name: string, fn: Transform): Transform;
 export declare function make_progressive_textures(filePath: string, _options: TextureResizeOptions, opts: ProgressiveOptions): Transform;
 /** Options for the {@link textureResize} function. */

package/dist/transforms/needle_progressive.js

@@ -14,14 +14,22 @@ import { compressPrimitive } from './needle_mesh_transform.js';
 import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
 export async function make_progressive(opts) {
     const file = opts.path;
-    return foreachGLTF(file, async (f) => {
+    let success = true;
+    await foreachGLTF(file, async (f) => {
+        // Should we stop if one file fails?
+        if (!success) {
+            return;
+        }
         // Don't re-process files that are already LODs
         if (isMeshLOD(f) || isImageLOD(f)) {
             console.log(`Progressive Transform: Ignore existing LOD file at ${f}`);
             return;
         }
-        await onProcess(f, opts);
+        const res = await onProcess(f, opts);
+        if (!res)
+            success = false;
     });
+    return success;
 }
 function registerExtensions(io) {
     registerDefaultExtensions(io);
@@ -39,10 +47,11 @@ function registerExtensions(io) {
 async function onProcess(file, opts) {
     if (!file.endsWith(".glb") && !file.endsWith(".gltf") && !file.endsWith(".vrm")) {
         console.log("ERR: file is not gltf/glb");
-        return;
+        return false;
     }
     if (!existsSync(file)) {
         console.log("ERR: file does not exist: " + file);
+        return false;
     }
     try {
         const io = new NodeIO();
@@ -68,23 +77,16 @@ async function onProcess(file, opts) {
                 transforms.push(make_progressive_textures(file, { size: [smallesTextureSize, smallesTextureSize] }, opts));
             }
             // TODO: progressive meshes aren't supported yet for VRMs
-            if (!file.endsWith(".vrm") && opts.config?.meshes?.lods !== false) {
+            const isVRM = file.endsWith(".vrm");
+            if (!isVRM && opts.config?.meshes?.lods !== false) {
                 transforms.push(make_progressive_meshes(file, {}, opts));
             }
-            else
+            else if (isVRM)
                 logger.warn("WARN: Progressive meshes are not supported for VRM files yet");
             transforms.push(sparse());
             logger.info("→ Transforming: " + file);
             await document.transform(...transforms);
-            let filename = file;
-            if (opts.debug === true) {
-                // get extension
-                const extension = filename.split('.').pop();
-                // remove extension
-                filename = filename.split('.').slice(0, -1).join('.');
-                // add new extension
-                filename = filename + ".debug." + extension;
-            }
+            const filename = file;
             await writeNodeIO(io, filename, document);
             opts.results.push(filename);
             logger.info("← Transforming done: " + filename);
@@ -92,7 +94,9 @@ async function onProcess(file, opts) {
     }
     catch (err) {
         console.log("ERR: failed to process file \"" + file + "\", Reason: " + err.message, "\nStack:\n" + err.stack);
+        return false;
     }
+    return true;
 }
 // basic idea:
 // 1. iterate all images that have an extension marking them to be progressively loaded
@@ -168,11 +172,15 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 logger.debug(`[${NAME}] Skipping because progressive extension is not available. ${textureIndex}`);
                 continue;
             }
+            /** This is the current texture's size  */
+            const currentSize = Math.min(...texture.getSize());
+            if (currentSize <= settings.maxSize) {
+                logger.debug(`[${NAME}] Skipping, texture[${textureIndex}] is already smaller than the requested size: ${currentSize}px <= ${settings.maxSize}px`);
+                continue;
+            }
             const textureGuid = settings.guid?.length ? settings.guid : generateGuid(filename + "_texture_" + index.toString());
             /** This is the size of the texture that is embedded in the glTF. No LOD should be smaller than this */
             const maxSize = settings.maxSize;
-            /** This is the current texture's size  */
-            const currentSize = Math.min(...texture.getSize());
             const LOD_LEVEL = new Array();
             LOD_LEVEL.push({ size: currentSize });
             let generateLods = true; //opts.config?.usecase === "world";
@@ -309,27 +317,32 @@ async function createTextureLod(args, opts) {
         // Make sure the material is used
         createMeshForMaterial(newDoc, newMaterial, slot_index++);
     }
-    const compressionSettings = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
-    if (compressionSettings !== undefined && compressionSettings !== null && compressionSettings.getExtensionDefinition) {
+    const compressionSettingsExt = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
+    if (compressionSettingsExt !== undefined && compressionSettingsExt !== null && compressionSettingsExt.getExtensionDefinition) {
         logger.info("> Pass compression extension to new texture");
         const newExt = newDoc.createExtension(NEEDLE_compression_texture);
-        const def = compressionSettings.getExtensionDefinition();
-        switch (opts.config.usecase) {
-            // For world scenarios we use ETC1S
-            case "world":
-                def.mode = "ETC1S";
-                break;
-            case "product":
-                def.mode = detectTextureCompressionMode(slots);
-                def.quality = 100;
-                break;
-            default:
-                def.mode = detectTextureCompressionMode(slots);
-                break;
+        const compressionSettings = compressionSettingsExt.getExtensionDefinition();
+        if (compressionSettings?.mode === "none") {
+            // if the mode is set to None then we keep that setting for the new texture
+        }
+        else {
+            switch (opts.config.usecase) {
+                // For world scenarios we use ETC1S
+                case "world":
+                    compressionSettings.mode = "ETC1S";
+                    break;
+                case "product":
+                    compressionSettings.mode = detectTextureCompressionMode(slots);
+                    compressionSettings.quality = 100;
+                    break;
+                default:
+                    compressionSettings.mode = detectTextureCompressionMode(slots);
+                    break;
+            }
         }
-        def.slot = slots[0];
-        const newProp = newExt.create(def);
-        newTexture.setExtension(NEEDLE_compression_texture.EXTENSION_NAME, newProp);
+        compressionSettings.slot = slots[0];
+        const newCompressionSettings = newExt.create(compressionSettings);
+        newTexture.setExtension(NEEDLE_compression_texture.EXTENSION_NAME, newCompressionSettings);
     }
     // add progressive texture extension
     const progressiveExt = newDoc.createExtension(NEEDLE_progressive);
@@ -596,7 +609,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
                                 shouldOmitLevel = true;
                             }
                             else if (vertexDifference < 32 || diff01 < 0.05) {
-                                console.log(`[${NAME}] Flag mesh LOD ${level}: vertex count difference below threshold (removed ${vertexDifference.toFixed(0)} vertices, ${(-diff01 * 100).toFixed(4)}%) → LOD ${level} will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
+                                logger.warn(`[${NAME}] Flag mesh LOD ${level}: vertex count difference below threshold (removed ${vertexDifference.toFixed(0)} vertices, ${(-diff01 * 100).toFixed(4)}%) → LOD ${level} will be omitted. \"${mesh.getName()}\" at \"${lodInfo.path}\"`);
                                 shouldOmitLevel = true;
                             }
                         }
@@ -642,7 +655,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
             }
             else if (difference < 32 || diff01 <= 0.05) {
                 // the lowest LOD has the same vertex count as the original mesh → we don't need to write the extension and can also delete previously generated LODs
-                logger.info(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length} (embedded): vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove LOD. \"${mesh.getName()}\"`);
+                logger.info(`[${NAME}]: Flag mesh LOD ${generatedLODInfos.length} (embedded): vertex count difference below threshold (removed ${difference} vertices, ${(-diff01 * 100).toFixed(2)} %) → will remove all LODs. \"${mesh.getName()}\"`);
                 try {
                     for (const lodInfo of generatedLODInfos) {
                         rmSync(outputDirectory + "/" + lodInfo.path);
@@ -695,7 +708,8 @@ function clonePrimitiveData(prim, logger) {
     if (semantics.some(s => s.includes("JOINTS"))) {
         return null;
     }
-    const indices = new Uint32Array(prim.getIndices().getArray());
+    const prevIndices = prim.getIndices();
+    const indices = !prevIndices ? new Uint32Array() : new Uint32Array(prevIndices.getArray());
     const attributes = {};
     for (const attr of semantics) {
         logger.info(`[${NAME}] Clone attribute ${attr}`);
@@ -760,7 +774,7 @@ async function createMeshLod(doc, basePath, level, mesh, meshInfo, guid, lodLeve
     newNode.setName("LOD " + level + "_" + guid);
     newNode.setTranslation([0, 0, 0]);
     const newMesh = newDoc.createMesh();
-    newMesh.setName(mesh.getName() + " LOD " + level);
+    newMesh.setName((mesh.getName() || "Unnamed") + " LOD " + level);
     newNode.setMesh(newMesh);
     const newMat = newDoc.createMaterial();
     const weights = mesh.getWeights();
@@ -882,6 +896,8 @@ async function simplifyMesh(doc, mesh, prim, index, lodinfo) {
     const weldOptions = {
         ...WELD_DEFAULTS,
         overwrite: true,
+        cleanup: false,
+        tolerance: 0.00000000001,
         ...lodinfo,
     };
     const simplifyOptions = {

package/dist/transforms/needle_texture_transform.js

@@ -39,8 +39,12 @@ export const needle_texture_transform = function (options) {
     };
     return createTransform("needle_texture_transform", async (document) => {
         await webPContext.prepare(document);
-        if (context.noToktx !== true)
-            await toktxContext.prepare(document);
+        if (context.noToktx !== true) {
+            const validToktx = await toktxContext.prepare(document);
+            if (!validToktx) {
+                context.noToktx = true;
+            }
+        }
         try {
             const logger = document.getLogger();
             const textures = document.getRoot().listTextures();
@@ -168,7 +172,7 @@ async function processTexture(texture, index, document, logger, context) {
             successFullyProcessed = await context.toktxContext.process(index, texture, options, context);
         }
         else {
-            console.log("Toktx not found: Skipping texture compression.");
+            console.log("Toktx not found: Skipping texture compression: " + originalMimeType);
         }
     }
     if (context.useCache && successFullyProcessed && key) {

package/dist/transforms/needle_toktx.js

@@ -1,7 +1,7 @@
 import { BufferUtils, FileUtils, ImageUtils, uuid } from "@gltf-transform/core";
 import { listTextureSlots, getTextureChannelMask, fitPowerOfTwo, TextureResizeFilter } from "@gltf-transform/functions";
 import { KHRTextureBasisu } from "@gltf-transform/extensions";
-import { checkKTXSoftware, createParams, Mode } from "./toktx.js";
+import { checkKTXSoftware, createParams, KTX_SOFTWARE_VERSION_MIN, Mode } from "./toktx.js";
 import { existsSync, mkdirSync } from "fs";
 import { formatBytes, spawn, waitExit } from './util.js';
 import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "./toktx_types.js";
@@ -70,7 +70,7 @@ export class NeedleToktxContext {
             const logger = document.getLogger();
             this.version = await checkKTXSoftware(logger, this.env);
             if (this.version === null) {
-                console.log("Will not run toktx transform");
+                logger.warn(`WARN: toktx not found, skipping texture compression. Please install KTX-Software >= ${KTX_SOFTWARE_VERSION_MIN}`);
                 this._prepareFailed = true;
                 return false;
             }

package/dist/transforms/toktx.d.ts

@@ -1,5 +1,6 @@
 import { Document, ILogger, vec2, Texture } from '@gltf-transform/core';
 import { ETC1SOptions, UASTCOptions } from './toktx_types.js';
+export declare const KTX_SOFTWARE_VERSION_MIN = "4.3.0";
 export declare const Filter: {
     BOX: string;
     TENT: string;

package/dist/transforms/toktx.js

@@ -10,8 +10,7 @@ import { TextureResizeFilter, getTextureColorSpace } from '@gltf-transform/funct
 import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from './toktx_types.js';
 tmp.setGracefulCleanup();
 const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
-const KTX_SOFTWARE_VERSION_MIN = '4.0.0-rc1';
-const KTX_SOFTWARE_VERSION_ACTIVE = '4.1.0-rc1';
+export const KTX_SOFTWARE_VERSION_MIN = '4.3.0';
 export const Filter = {
     BOX: 'box',
     TENT: 'tent',
@@ -167,6 +166,7 @@ export async function checkKTXSoftware(logger, env) {
     }
     else if (semver.lt(semver.clean(version), KTX_SOFTWARE_VERSION_MIN)) {
         logger.warn(`toktx: Expected KTX-Software >= v${KTX_SOFTWARE_VERSION_MIN}, found ${version}.`);
+        return null;
     }
     else {
         if (!didLogToktxVersion)

package/dist/utils/mesh.d.ts

@@ -5,4 +5,9 @@ export declare type MeshInformation = {
     readonly totalIndexCount: number;
 };
 export declare function getMeshInformation(mesh: Mesh): MeshInformation;
-export declare function calculateMeshDensity(prim: Primitive): number;
+/**
+ * Calculate the density of a mesh.
+ * @param prim - The primitive to calculate the density for.
+ * @returns The density of the mesh or -1 if the mesh is not indexed.
+ */
+export declare function calculateMeshDensity(prim: Primitive): number | -1;

package/dist/utils/mesh.js

@@ -14,6 +14,11 @@ export function getMeshInformation(mesh) {
         totalIndexCount,
     };
 }
+/**
+ * Calculate the density of a mesh.
+ * @param prim - The primitive to calculate the density for.
+ * @returns The density of the mesh or -1 if the mesh is not indexed.
+ */
 export function calculateMeshDensity(prim) {
     const bounds = {
         min: [Infinity, Infinity, Infinity],
@@ -71,16 +76,28 @@ export function calculateMeshDensity(prim) {
             // Add the area (half the magnitude of the cross product) to the total
             area += Math.sqrt(cross[0] * cross[0] + cross[1] * cross[1] + cross[2] * cross[2]) / 2;
         }
+        const count = indices.getCount() / 3;
+        const density = count / area;
+        // const size = [0, 0, 0] as vec3;
+        // size[0] = bounds.max[0] - bounds.min[0];
+        // size[1] = bounds.max[1] - bounds.min[1];
+        // size[2] = bounds.max[2] - bounds.min[2];
+        // const volume = size[0] * size[1] * size[2];
+        // const density = count / volume;
+        return density;
     }
-    const count = indices.getCount() / 3;
-    const density = count / area;
-    // const size = [0, 0, 0] as vec3;
-    // size[0] = bounds.max[0] - bounds.min[0];
-    // size[1] = bounds.max[1] - bounds.min[1];
-    // size[2] = bounds.max[2] - bounds.min[2];
-    // const volume = size[0] * size[1] * size[2];
-    // const density = count / volume;
-    return density;
+    else {
+        // calculate density if the mesh is not indexed
+        const count = position.getCount();
+        const size = [0, 0, 0];
+        size[0] = bounds.max[0] - bounds.min[0];
+        size[1] = bounds.max[1] - bounds.min[1];
+        size[2] = bounds.max[2] - bounds.min[2];
+        const volume = size[0] * size[1] * size[2];
+        const density = count / volume;
+        return density;
+    }
+    return -1;
     function expandBounds(point, target) {
         for (let i = 0; i < 3; i++) {
             target.min[i] = Math.min(point[i], target.min[i]);

package/dist/utils/stats.js

@@ -4,7 +4,7 @@ import { statSync, writeFileSync } from "fs";
 export function printStats(stats, logger, config) {
     const duration = Date.now() - stats.startTime;
     const durationInMinutes = duration / 1000 / 60;
-    const msg = `[Needle Build Pipeline] v${getVersion()} — Finished ${config?.usecase} compression ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
+    const msg = `[Needle Build Pipeline] v${getVersion()} — Finished compression \"${config?.usecase}\" of ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
     if (logger)
         logger.info(msg);
     else

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.2.0-alpha.1",
+  "version": "2.2.0-alpha.11",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
-  "main": "src/index.js",
+  "main": "dist/index.js",
   "type": "module",
   "readme": "README.md",
   "author": {

package/scripts/clear-caches.mjs

@@ -1,4 +1,4 @@
 
-import { clearCache } from '../dist/src/cache/index.js';
+import { clearCache } from '../dist/cache/index.js';
 
 clearCache();

package/scripts/defaults.mjs

@@ -1,6 +1,6 @@
 import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
-import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS } from "../dist/src/extensions/NEEDLE_opaque/index.js";
-import { NEEDLE_progressive } from '../dist/src/index.js';
+import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS } from "../dist/extensions/NEEDLE_opaque/index.js";
+import { NEEDLE_progressive } from '../dist/index.js';
 
 
 export function registerExtensions(node) {

package/scripts/fileutils.mjs

@@ -1,4 +1,4 @@
-import { foreachGLTF as foreach } from '../dist/src/index.js';
+import { foreachGLTF as foreach } from '../dist/index.js';
 
 
 export function foreachGLTF(args, callback) {

package/scripts/index.mjs

@@ -1,6 +1,6 @@
 
 import { spawnSync } from "child_process";
-import { getVersion } from "../dist/src/index.js";
+import { getVersion } from "../dist/index.js";
 
 const workingDir = process.cwd();
 

package/scripts/limit-caches.mjs

@@ -1,4 +1,4 @@
 
-import { limitCacheSize } from '../dist/src/cache/index.js';
+import { limitCacheSize } from '../dist/cache/index.js';
 
 limitCacheSize(512);

package/scripts/make-progressive.mjs

@@ -1,5 +1,5 @@
-import { getConfig, getVersion } from '../dist/src/index.js';
-import { make_progressive } from '../dist/src/transforms/needle_progressive.js';
+import { getConfig, getVersion } from '../dist/index.js';
+import { make_progressive } from '../dist/transforms/needle_progressive.js';
 
 const args = process.argv;
 

package/scripts/pack-gltf.mjs

@@ -1,6 +1,6 @@
 import commandLineArgs from "command-line-args";
 import { foreachGLTF } from './fileutils.mjs';
-import { getVersion, limitCacheSize, getConfig, packGLTF, printStats } from '../dist/src/index.js';
+import { getVersion, limitCacheSize, getConfig, packGLTF, printStats } from '../dist/index.js';
 import { Logger, Verbosity } from "@gltf-transform/core";
 
 console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`)
@@ -60,7 +60,7 @@ if (noToktx)
 const packedFiles = new Set();
 
 /**
- * @type {import("../dist/src/index.js").Stats}
+ * @type {import("../dist/index.js").Stats}
  */
 const stats = {
     startTime: Date.now(),