@needle-tools/gltf-build-pipeline 2.15.11 → 2.16.0-alpha.bc3af72

package/dist/cache/cache.d.ts

@@ -54,3 +54,5 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
  * @returns The hash as a number.
  */
 export declare function getHash(property: ExtensibleProperty | object, level?: number): string;
+export declare function hashBuffer(buffer: ArrayBufferLike, offset?: number, length?: number): number;
+export declare function hashString(str: string): number;

package/dist/cache/cache.js

@@ -307,7 +307,7 @@ function hashObject(obj, depth = 0) {
         return fallbackHash;
     }
 }
-function hashBuffer(buffer, offset, length) {
+export function hashBuffer(buffer, offset, length) {
     if (xxHashModule) {
         const view = new Uint8Array(buffer, offset, length);
         return xxHashModule.h32Raw(view);
@@ -320,7 +320,7 @@ function hashBuffer(buffer, offset, length) {
     }
     return hash;
 }
-function hashString(str) {
+export function hashString(str) {
     let hash = 0;
     for (let i = 0; i < str.length; i++) {
         hash = ((hash << 5) - hash) + str.charCodeAt(i);

package/dist/cache/index.d.ts

@@ -1 +1,2 @@
 export * from "./cache.js";
+export * from "./job-cache.js";

package/dist/cache/index.js

@@ -1 +1,2 @@
 export * from "./cache.js";
+export * from "./job-cache.js";

package/dist/cache/job-cache.d.ts

@@ -0,0 +1,28 @@
+import { ILogger } from '@gltf-transform/core';
+import { Config } from '../config/index.js';
+interface JobCacheOptions {
+    compress: boolean;
+    progressive: boolean;
+    config: Config;
+}
+/**
+ * Compute a job cache key from pipeline options and input file content.
+ * The key uniquely identifies a specific input + configuration combination.
+ * Currently only supports .glb files (hashes the full file content).
+ * @returns The job key string, or null if the file format is not supported.
+ */
+export declare function computeJobKey(options: JobCacheOptions, inputFilePath: string): string | null;
+/**
+ * Save the result of a completed job to the cache.
+ * Stores the content of each output file and a manifest linking them.
+ * The caller should include all output files including external sidecar files
+ * (e.g. .pmrem.ktx2) collected via PackGLTFOptions.externalFiles.
+ */
+export declare function saveJobResult(jobKey: string, outputFiles: string[], logger: ILogger): void;
+/**
+ * Try to restore a previously cached job result.
+ * Loads all cached files into memory first, then writes them only if all are available.
+ * Returns the list of restored file paths, or null if the cache is incomplete.
+ */
+export declare function tryRestoreJobResult(jobKey: string, outputDir: string, logger: ILogger): string[] | null;
+export {};

package/dist/cache/job-cache.js

@@ -0,0 +1,86 @@
+// TODO: avoid storing LOD files twice in cache. Currently LODs are stored by the per-asset cache
+// (e.g. "image-{hash}", "packed-{hash}") AND again by the job cache ("job-{key}-fN").
+// Instead, progressive_results should carry { path, cacheKey? } so the job manifest can reference
+// existing per-asset cache keys for LODs and only store the main GLB under a job-specific key.
+// This requires threading cache key collection through make_progressive and packGLTF.
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import path from 'path';
+import { addMetaToCache, addToCache, tryGetFromCache, tryGetMetaFromCache, hashBuffer, hashString } from './cache.js';
+/**
+ * Compute a job cache key from pipeline options and input file content.
+ * The key uniquely identifies a specific input + configuration combination.
+ * Currently only supports .glb files (hashes the full file content).
+ * @returns The job key string, or null if the file format is not supported.
+ */
+// TODO: add support for .gltf files (need to hash referenced .bin files as well)
+export function computeJobKey(options, inputFilePath) {
+    if (!inputFilePath.endsWith('.glb'))
+        return null;
+    const fileBytes = readFileSync(inputFilePath);
+    const fileHash = hashBuffer(fileBytes.buffer, fileBytes.byteOffset, fileBytes.byteLength);
+    const optionsStr = JSON.stringify({
+        compress: options.compress,
+        progressive: options.progressive,
+        config: options.config,
+    });
+    const optionsHash = hashString(optionsStr);
+    return `job-${optionsHash}-${fileHash}`;
+}
+/**
+ * Save the result of a completed job to the cache.
+ * Stores the content of each output file and a manifest linking them.
+ * The caller should include all output files including external sidecar files
+ * (e.g. .pmrem.ktx2) collected via PackGLTFOptions.externalFiles.
+ */
+export function saveJobResult(jobKey, outputFiles, logger) {
+    const entries = [];
+    for (let i = 0; i < outputFiles.length; i++) {
+        const filePath = outputFiles[i];
+        if (!existsSync(filePath))
+            continue;
+        const cacheKey = `${jobKey}-f${i}`;
+        const content = readFileSync(filePath);
+        addToCache(cacheKey, content);
+        entries.push({
+            fileName: path.basename(filePath),
+            cacheKey,
+        });
+    }
+    if (entries.length === 0)
+        return;
+    const manifest = { entries };
+    addMetaToCache(jobKey, manifest);
+    logger.debug(`[Job Cache] Saved ${entries.length} file(s) for job ${jobKey}`);
+}
+/**
+ * Try to restore a previously cached job result.
+ * Loads all cached files into memory first, then writes them only if all are available.
+ * Returns the list of restored file paths, or null if the cache is incomplete.
+ */
+export function tryRestoreJobResult(jobKey, outputDir, logger) {
+    const manifest = tryGetMetaFromCache(jobKey);
+    if (!manifest?.entries?.length)
+        return null;
+    // Load all entries into memory — fail fast if any are missing
+    const loaded = [];
+    for (const entry of manifest.entries) {
+        const cached = tryGetFromCache(entry.cacheKey);
+        if (!cached) {
+            logger.debug(`[Job Cache] Cache miss: missing entry ${entry.cacheKey}`);
+            return null;
+        }
+        loaded.push({ entry, data: cached });
+    }
+    // All entries present — write to output directory
+    if (!existsSync(outputDir)) {
+        mkdirSync(outputDir, { recursive: true });
+    }
+    const restoredPaths = [];
+    for (const { entry, data } of loaded) {
+        const outPath = path.join(outputDir, entry.fileName);
+        writeFileSync(outPath, data);
+        restoredPaths.push(outPath);
+    }
+    logger.info(`[Job Cache] Restored ${restoredPaths.length} file(s) from cache`);
+    return restoredPaths;
+}

package/dist/cli/index.js

@@ -6,9 +6,10 @@ import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { isLOD, make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
-import { existsSync } from 'fs';
+import { computeJobKey, saveJobResult, tryRestoreJobResult } from '../cache/job-cache.js';
+import { existsSync, statSync } from 'fs';
 import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
-import path, { resolve } from 'path';
+import path, { dirname, resolve } from 'path';
 import { ERROR_CODES } from '../constants.js';
 import { trackPipelineStart, trackPipelineEnd, trackError } from '../utils/analytics.js';
 // For testing / dev you can run `npm link` in the package directory
@@ -52,6 +53,7 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
     }
     logger.info("Writing stats to: " + output);
     writeFileStatsToFile(filestats, output);
+    process.exit(0);
 })
     .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
     .help(`
@@ -140,6 +142,32 @@ Each version will be compressed and written to the output directory.
             logger.debug(`→ Ignore existing LOD file at ${file}`);
             return;
         }
+        // Job-level cache: skip all work if the same input + options were already processed
+        const jobKey = useCache ? computeJobKey({
+            compress: options.compress === true,
+            progressive: options.progressive === true,
+            config,
+        }, file) : undefined;
+        if (jobKey) {
+            const outputDir = output
+                ? (existsSync(output) && isDirectory(output) ? output : dirname(output))
+                : dirname(file);
+            const restored = tryRestoreJobResult(jobKey, outputDir, logger);
+            if (restored) {
+                const inputFileSize = statSync(file).size / 1024 / 1024;
+                for (const f of restored) {
+                    set.add(f);
+                    if (stats) {
+                        const sizeInMB = statSync(f).size / 1024 / 1024;
+                        stats.totalFilesProcessed++;
+                        stats.totalFileSizeInMB += sizeInMB;
+                        stats.totalFileSizeInMBBefore += inputFileSize;
+                    }
+                }
+                logger.info(`→ [CACHE] Restored ${restored.length} file(s)for job ${jobKey}`);
+                return;
+            }
+        }
         logger.info(`→ Transform ${file}`);
         const progressive_results = new Array();
         if (options.progressive === true) {
@@ -164,6 +192,7 @@ Each version will be compressed and written to the output directory.
                 console.log("Progressive results:\n", progressive_results);
             }
         }
+        const externalFiles = [];
         if (options.compress === true) {
             const opts = {
                 config,
@@ -176,6 +205,7 @@ Each version will be compressed and written to the output directory.
                 // When progressive ran first, inputFile points to the progressive output copy.
                 // Pass the original source path so extensions can resolve sibling assets (e.g. .exr files).
                 sourceFile: progressive_results.length > 0 ? file : undefined,
+                externalFiles,
             };
             // If we have produced progressive assets then the array already contains the input
             if (progressive_results.length > 0) {
@@ -195,6 +225,13 @@ Each version will be compressed and written to the output directory.
                 }
             }
         }
+        // Save job result to cache for future runs
+        if (jobKey) {
+            const allOutputFiles = progressive_results.length > 0
+                ? progressive_results
+                : [output || file];
+            saveJobResult(jobKey, [...allOutputFiles, ...externalFiles], logger);
+        }
     });
     printStats(stats, logger, config);
     trackPipelineEnd({

package/dist/scripts/pack-gltf.d.ts

@@ -11,6 +11,8 @@ export type PackGLTFOptions = {
     verbose?: boolean;
     /** Original source file path, used to resolve external assets (e.g. .exr files) when inputFile differs from the source (e.g. after progressive transform) */
     sourceFile?: string;
+    /** Populated by packGLTF with absolute paths to external sidecar files produced during processing (e.g. .pmrem.ktx2) */
+    externalFiles?: string[];
 };
 /** Compress a glTF file
  */

package/dist/scripts/pack-gltf.js

@@ -7,7 +7,7 @@ import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 import { dedup, metalRough, prune, resample } from '@gltf-transform/functions';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, setExtensionInputFile } from '../extensions/index.js';
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
-import { copyExternalResources, getOutputPath, getVersion, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
+import { copyExternalResources, getOutputPath, getVersion, isMaterialOnlyGLB, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
 import { TestAssertionError } from '../utils/test-assert.js';
 /** Compress a glTF file
@@ -105,8 +105,11 @@ export async function packGLTF(inputFile, outputFile, options) {
             dedup({ propertyTypes: [PropertyType.MESH, PropertyType.MATERIAL] }),
         ];
         const isProgressiveAsset = isLOD(inputFile);
-        // TODO: progressive mesh LODs should not be pruned right now because it will remove e.g. UVs from meshes (and probably blend shapes too) (so we just not do it for any progressive asset) 
-        if (!isProgressiveAsset) {
+        const materialOnly = isMaterialOnlyGLB(document);
+        // TODO: progressive mesh LODs should not be pruned right now because it will remove e.g. UVs from meshes (and probably blend shapes too) (so we just not do it for any progressive asset)
+        // Material-only GLBs should not be pruned at all — they have no meshes so
+        // prune would strip materials, textures, and buffers, producing an empty file.
+        if (!isProgressiveAsset && !materialOnly) {
             transforms.push(prune({
                 // Pruning animations is not supported yet (e.g. if the animation is only referenced by a component)
                 propertyTypes: [
@@ -123,6 +126,29 @@ export async function packGLTF(inputFile, outputFile, options) {
             }));
         }
         await document.transform(...transforms);
+        // Collect external sidecar files (e.g. .pmrem.ktx2) produced by transforms
+        if (options.externalFiles) {
+            const outDir = path.dirname(outputFile);
+            const root = document.getRoot();
+            for (const tex of root.listTextures()) {
+                const uri = tex.getURI();
+                if (uri && !uri.startsWith('data:')) {
+                    const absPath = path.resolve(outDir, uri);
+                    if (existsSync(absPath)) {
+                        options.externalFiles.push(absPath);
+                    }
+                }
+            }
+            for (const buf of root.listBuffers()) {
+                const uri = buf.getURI();
+                if (uri && !uri.startsWith('data:')) {
+                    const absPath = path.resolve(outDir, uri);
+                    if (existsSync(absPath)) {
+                        options.externalFiles.push(absPath);
+                    }
+                }
+            }
+        }
         logger.debug(`← Writing to ${outputFile}`);
         await writeNodeIO(io, outputFile, document);
         if (useCache && cacheKey != undefined && existsSync(outputFile))

package/dist/utils/validate.d.ts

@@ -1,4 +1,7 @@
-import { ILogger } from "@gltf-transform/core";
+import { Document, ILogger } from "@gltf-transform/core";
+/** Returns true when the document contains materials/textures but no meshes, nodes, or scenes.
+ * These are standalone material assets that should not be pruned. */
+export declare function isMaterialOnlyGLB(document: Document): boolean;
 export declare function validateImageMimeType(mimetype: string, image: Uint8Array, logger: ILogger): {
     error: boolean;
     message: string;

package/dist/utils/validate.js

@@ -1,4 +1,11 @@
 import { ImageUtils } from "@gltf-transform/core";
+/** Returns true when the document contains materials/textures but no meshes, nodes, or scenes.
+ * These are standalone material assets that should not be pruned. */
+export function isMaterialOnlyGLB(document) {
+    const root = document.getRoot();
+    return root.listMeshes().length === 0
+        && root.listMaterials().length > 0;
+}
 export function validateImageMimeType(mimetype, image, logger) {
     const actualMimeType = ImageUtils.getMimeType(image);
     if (actualMimeType != null && actualMimeType != mimetype) {

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.15.11";
+export declare const version = "2.16.0-alpha.bc3af72";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.15.11";
+export const version = "2.16.0-alpha.bc3af72";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.15.11",
+  "version": "2.16.0-alpha.bc3af72",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",