@needle-tools/gltf-build-pipeline 2.15.11 → 2.16.0-next.fc19d36

package/dist/cache/cache.d.ts

@@ -54,3 +54,5 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
  * @returns The hash as a number.
  */
 export declare function getHash(property: ExtensibleProperty | object, level?: number): string;
+export declare function hashBuffer(buffer: ArrayBufferLike, offset?: number, length?: number): number;
+export declare function hashString(str: string): number;

package/dist/cache/cache.js

@@ -307,7 +307,7 @@ function hashObject(obj, depth = 0) {
         return fallbackHash;
     }
 }
-function hashBuffer(buffer, offset, length) {
+export function hashBuffer(buffer, offset, length) {
     if (xxHashModule) {
         const view = new Uint8Array(buffer, offset, length);
         return xxHashModule.h32Raw(view);
@@ -320,7 +320,7 @@ function hashBuffer(buffer, offset, length) {
     }
     return hash;
 }
-function hashString(str) {
+export function hashString(str) {
     let hash = 0;
     for (let i = 0; i < str.length; i++) {
         hash = ((hash << 5) - hash) + str.charCodeAt(i);

package/dist/cache/index.d.ts

@@ -1 +1,2 @@
 export * from "./cache.js";
+export * from "./job-cache.js";

package/dist/cache/index.js

@@ -1 +1,2 @@
 export * from "./cache.js";
+export * from "./job-cache.js";

package/dist/cache/job-cache.d.ts

@@ -0,0 +1,26 @@
+import { ILogger } from '@gltf-transform/core';
+import { Config } from '../config/index.js';
+interface JobCacheOptions {
+    compress: boolean;
+    progressive: boolean;
+    config: Config;
+}
+/**
+ * Compute a job cache key from pipeline options and input file content.
+ * The key uniquely identifies a specific input + configuration combination.
+ * Currently only supports .glb files (hashes the full file content).
+ * @returns The job key string, or null if the file format is not supported.
+ */
+export declare function computeJobKey(options: JobCacheOptions, inputFilePath: string): string | null;
+/**
+ * Save the result of a completed job to the cache.
+ * Stores the content of each output file and a manifest linking them.
+ */
+export declare function saveJobResult(jobKey: string, outputFiles: string[], logger: ILogger): void;
+/**
+ * Try to restore a previously cached job result.
+ * Loads all cached files into memory first, then writes them only if all are available.
+ * Returns the list of restored file paths, or null if the cache is incomplete.
+ */
+export declare function tryRestoreJobResult(jobKey: string, outputDir: string, logger: ILogger): string[] | null;
+export {};

package/dist/cache/job-cache.js

@@ -0,0 +1,84 @@
+// TODO: avoid storing LOD files twice in cache. Currently LODs are stored by the per-asset cache
+// (e.g. "image-{hash}", "packed-{hash}") AND again by the job cache ("job-{key}-fN").
+// Instead, progressive_results should carry { path, cacheKey? } so the job manifest can reference
+// existing per-asset cache keys for LODs and only store the main GLB under a job-specific key.
+// This requires threading cache key collection through make_progressive and packGLTF.
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import path from 'path';
+import { addMetaToCache, addToCache, tryGetFromCache, tryGetMetaFromCache, hashBuffer, hashString } from './cache.js';
+/**
+ * Compute a job cache key from pipeline options and input file content.
+ * The key uniquely identifies a specific input + configuration combination.
+ * Currently only supports .glb files (hashes the full file content).
+ * @returns The job key string, or null if the file format is not supported.
+ */
+// TODO: add support for .gltf files (need to hash referenced .bin files as well)
+export function computeJobKey(options, inputFilePath) {
+    if (!inputFilePath.endsWith('.glb'))
+        return null;
+    const fileBytes = readFileSync(inputFilePath);
+    const fileHash = hashBuffer(fileBytes.buffer, fileBytes.byteOffset, fileBytes.byteLength);
+    const optionsStr = JSON.stringify({
+        compress: options.compress,
+        progressive: options.progressive,
+        config: options.config,
+    });
+    const optionsHash = hashString(optionsStr);
+    return `job-${optionsHash}-${fileHash}`;
+}
+/**
+ * Save the result of a completed job to the cache.
+ * Stores the content of each output file and a manifest linking them.
+ */
+export function saveJobResult(jobKey, outputFiles, logger) {
+    const entries = [];
+    for (let i = 0; i < outputFiles.length; i++) {
+        const filePath = outputFiles[i];
+        if (!existsSync(filePath))
+            continue;
+        const cacheKey = `${jobKey}-f${i}`;
+        const content = readFileSync(filePath);
+        addToCache(cacheKey, content);
+        entries.push({
+            fileName: path.basename(filePath),
+            cacheKey,
+        });
+    }
+    if (entries.length === 0)
+        return;
+    const manifest = { entries };
+    addMetaToCache(jobKey, manifest);
+    logger.debug(`[Job Cache] Saved ${entries.length} file(s) for job ${jobKey}`);
+}
+/**
+ * Try to restore a previously cached job result.
+ * Loads all cached files into memory first, then writes them only if all are available.
+ * Returns the list of restored file paths, or null if the cache is incomplete.
+ */
+export function tryRestoreJobResult(jobKey, outputDir, logger) {
+    const manifest = tryGetMetaFromCache(jobKey);
+    if (!manifest?.entries?.length)
+        return null;
+    // Load all entries into memory — fail fast if any are missing
+    const loaded = [];
+    for (const entry of manifest.entries) {
+        const cached = tryGetFromCache(entry.cacheKey);
+        if (!cached) {
+            logger.debug(`[Job Cache] Cache miss: missing entry ${entry.cacheKey}`);
+            return null;
+        }
+        loaded.push({ entry, data: cached });
+    }
+    // All entries present — write to output directory
+    if (!existsSync(outputDir)) {
+        mkdirSync(outputDir, { recursive: true });
+    }
+    const restoredPaths = [];
+    for (const { entry, data } of loaded) {
+        const outPath = path.join(outputDir, entry.fileName);
+        writeFileSync(outPath, data);
+        restoredPaths.push(outPath);
+    }
+    logger.info(`[Job Cache] Restored ${restoredPaths.length} file(s) from cache`);
+    return restoredPaths;
+}

package/dist/cli/index.js

@@ -6,9 +6,10 @@ import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { isLOD, make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
-import { existsSync } from 'fs';
+import { computeJobKey, saveJobResult, tryRestoreJobResult } from '../cache/job-cache.js';
+import { existsSync, statSync } from 'fs';
 import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
-import path, { resolve } from 'path';
+import path, { dirname, resolve } from 'path';
 import { ERROR_CODES } from '../constants.js';
 import { trackPipelineStart, trackPipelineEnd, trackError } from '../utils/analytics.js';
 // For testing / dev you can run `npm link` in the package directory
@@ -140,6 +141,32 @@ Each version will be compressed and written to the output directory.
             logger.debug(`→ Ignore existing LOD file at ${file}`);
             return;
         }
+        // Job-level cache: skip all work if the same input + options were already processed
+        const jobKey = useCache ? computeJobKey({
+            compress: options.compress === true,
+            progressive: options.progressive === true,
+            config,
+        }, file) : undefined;
+        if (jobKey) {
+            const outputDir = output
+                ? (existsSync(output) && isDirectory(output) ? output : dirname(output))
+                : dirname(file);
+            const restored = tryRestoreJobResult(jobKey, outputDir, logger);
+            if (restored) {
+                const inputFileSize = statSync(file).size / 1024 / 1024;
+                for (const f of restored) {
+                    set.add(f);
+                    if (stats) {
+                        const sizeInMB = statSync(f).size / 1024 / 1024;
+                        stats.totalFilesProcessed++;
+                        stats.totalFileSizeInMB += sizeInMB;
+                        stats.totalFileSizeInMBBefore += inputFileSize;
+                    }
+                }
+                logger.info(`→ [CACHE] Restored ${restored.length} file(s)for job ${jobKey}`);
+                return;
+            }
+        }
         logger.info(`→ Transform ${file}`);
         const progressive_results = new Array();
         if (options.progressive === true) {
@@ -195,6 +222,13 @@ Each version will be compressed and written to the output directory.
                 }
             }
         }
+        // Save job result to cache for future runs
+        if (jobKey) {
+            const allOutputFiles = progressive_results.length > 0
+                ? progressive_results
+                : [output || file];
+            saveJobResult(jobKey, allOutputFiles, logger);
+        }
     });
     printStats(stats, logger, config);
     trackPipelineEnd({

package/dist/utils/version.gen.d.ts

@@ -1 +1 @@
-export declare const version = "2.15.11";
+export declare const version = "2.16.0-next.fc19d36";

package/dist/utils/version.gen.js

@@ -1 +1 @@
-export const version = "2.15.11";
+export const version = "2.16.0-next.fc19d36";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.15.11",
+  "version": "2.16.0-next.fc19d36",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",