@needle-tools/gltf-build-pipeline 2.13.0 → 2.13.1

package/dist/cache/cache.js

@@ -1,326 +1,326 @@
-import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
-import tmp from 'tmp';
-import fs, { statSync } from 'fs';
-import path from 'path';
-import { getVersion } from "../utils/version.js";
-// import * as CHECKDISCSPACE from 'check-disk-space'
-import xxhash from "xxhash-wasm";
-let hasher = null;
-xxhash().then(module => {
-    hasher = module.create32();
-});
-/**
- * Get the available space on the cache directory in bytes.
- */
-// export async function getCacheAvailableSpaceOnDisc(): Promise<number> {
-//     try {
-//         const dir = getCacheDirectory();
-//         const res = await CHECKDISCSPACE.default(dir).catch(() => { return { free: 0 } });
-//         return res.free;
-//     }
-//     catch (e) {
-//         console.error(`Error checking disk space: ${e}`);
-//     }
-//     return 0;
-// }
-const buildPipelineVersion = getVersion(false);
-const debugLogs = process.argv.includes("--debug");
-/**
- * Clear the cache.
- * This will remove all files in the cache directory.
- */
-export function clearCache(logger) {
-    const dir = getCacheDirectory(true);
-    fs.rmSync(dir, { recursive: true });
-    if (!logger)
-        console.log(`Cleared cache at ${dir}`);
-    else
-        logger.info(`Cleared cache at ${dir}`);
-}
-export const cacheSizeLimit = 1024 * 5;
-/**
- * Ensure that the cache size does not exceed the given maximum size in MB.
- * @param maxSizeMB The maximum size in MB.
- */
-export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
-    const dir = getCacheDirectory(true);
-    console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}`);
-    let totalSize = 0;
-    const files = new Array();
-    function recurseDirectories(dir) {
-        fs.readdirSync(dir).forEach(file => {
-            const path = `${dir}/${file}`;
-            const stats = statSync(path);
-            if (stats.isDirectory()) {
-                recurseDirectories(path);
-            }
-            else {
-                totalSize += stats.size;
-                files.push({ path, size: stats.size });
-            }
-        });
-    }
-    recurseDirectories(dir);
-    const maxSizeInBytes = maxSizeMB * 1024 * 1024;
-    console.log(`Current cache size is ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
-    if (totalSize < maxSizeInBytes)
-        return;
-    files.sort((a, b) => {
-        const mtimeA = fs.statSync(a.path).mtimeMs;
-        const mtimeB = fs.statSync(b.path).mtimeMs;
-        return mtimeA - mtimeB;
-    });
-    const targetSize = maxSizeInBytes * 0.9;
-    // remove oldest files first
-    let filesRemoved = 0;
-    while (totalSize > targetSize) {
-        const file = files.shift();
-        if (!file)
-            break;
-        if (fs.statSync(file.path).isDirectory()) {
-            continue;
-        }
-        filesRemoved++;
-        fs.rmSync(file.path);
-        totalSize -= file.size;
-    }
-    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB (${filesRemoved} files removed)`);
-}
-/**
- * Add data to the cache.
- * @param key The key to store the data under.
- * @param bytes The data to store as a Uint8Array.
- */
-export function addToCache(key, bytes) {
-    const dir = getCacheDirectory();
-    const path = `${dir}/${key}.bin`;
-    if (debugLogs)
-        console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
-    if (bytes instanceof Buffer)
-        bytes = new Uint8Array(bytes);
-    fs.writeFileSync(path, bytes);
-}
-/**
- * Try to get the data from the cache.
- * @param key The key to look up.
- * @returns The data as a Uint8Array or null if not found.
- * @throws If the cache file is corrupted.
- */
-export function tryGetFromCache(key) {
-    const dir = getCacheDirectory();
-    const path = `${dir}/${key}.bin`;
-    if (fs.existsSync(path)) {
-        const bytes = fs.readFileSync(path);
-        if (bytes instanceof Buffer)
-            return new Uint8Array(bytes);
-        return bytes;
-    }
-    return null;
-}
-/** Add meta data to the cache.
- * @param key The key to store the meta data for.
- * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
- */
-export function addMetaToCache(key, meta) {
-    const dir = getCacheDirectory();
-    const metaPath = `${dir}/${key}.meta`;
-    meta['needle-build-pipeline:version'] = buildPipelineVersion;
-    fs.writeFileSync(metaPath, JSON.stringify(meta));
-}
-/** Try to get the meta data from the cache.
- * @param key The key to look up.
- * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
- * @returns The meta data object or null if not found.
- */
-export function tryGetMetaFromCache(key, ignoreVersion = false) {
-    try {
-        const dir = getCacheDirectory();
-        const path = `${dir}/${key}.meta`;
-        if (fs.existsSync(path)) {
-            const bytes = fs.readFileSync(path);
-            if (bytes?.length > 0) {
-                const text = bytes.toString('utf-8');
-                if (text) {
-                    const obj = JSON.parse(text);
-                    if (ignoreVersion)
-                        return obj;
-                    if (obj['needle-build-pipeline:version'] === buildPipelineVersion)
-                        return obj;
-                }
-            }
-        }
-        return null;
-    }
-    catch (e) {
-        console.error(`Error reading cache meta: ${e.message}`);
-        return null;
-    }
-}
-/**
- * Get the cache key for a property.
- * @param property The property to get the key for.
- * @returns The key as a string.
- */
-export function getKey(property) {
-    let prefix = "prop";
-    if (property instanceof Texture) {
-        prefix = "image";
-    }
-    let hash = getHash(property);
-    const key = `${prefix}-${hash}`;
-    return key;
-}
-export function getKeyWithHash(property) {
-    let prefix = "prop";
-    if (property instanceof Texture) {
-        prefix = "image";
-    }
-    let hash = getHash(property);
-    const key = `${prefix}-${hash}`;
-    return { key, hash };
-}
-/**
- * Get the hash for a property or an object.
- * @param property The property to get the hash for.
- * @param level The current level of recursion.
- * @returns The hash as a number.
- */
-export function getHash(property, level = 0) {
-    let hashStr = "";
-    let hash = 0;
-    if (property instanceof ExtensibleProperty) {
-        const extensions = property.listExtensions();
-        const extras = property.getExtras();
-        // const slots = listTextureSlots(doc, texture);
-        // const channels = getTextureChannelMask(doc, texture);
-        hash += hashObject({
-            extensions,
-            extras,
-        });
-        if (property instanceof Texture) {
-            const image = property.getImage();
-            const bytes = image?.buffer;
-            if (bytes)
-                hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
-        }
-        else if (property instanceof Mesh) {
-            const weights = property.getWeights();
-            hash += hashObject(weights);
-            const primitives = property.listPrimitives();
-            for (const prim of primitives) {
-                hash += hashString(getHash(prim, level + 1));
-            }
-        }
-        else if (property instanceof Primitive) {
-            const indices = property.getIndices();
-            const attributes = property.listAttributes();
-            const targets = property.listTargets();
-            // TODO: hash buffer view if possible
-            const semantics = property.listSemantics();
-            const semanticsNames = semantics.map(s => {
-                return {
-                    name: s,
-                };
-            });
-            hash += hashObject({
-                indices,
-                attributes,
-                targets,
-                semantics: semanticsNames
-            });
-            if (indices) {
-                const indexBuffer = indices.getArray();
-                if (indexBuffer) {
-                    hash += hashBuffer(indexBuffer.buffer, indexBuffer.byteOffset, indexBuffer.byteLength);
-                }
-            }
-            // We can not pass the whole buffer into hashObject since very large meshes will cause errors when trying to stringify
-            // See https://linear.app/needle/issue/NE-6414
-            for (const sem of semantics) {
-                const arr = property.getAttribute(sem)?.getArray();
-                if (arr) {
-                    hash += hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength);
-                }
-            }
-        }
-    }
-    else {
-        hash += hashObject(property);
-    }
-    return hashStr + hash;
-}
-const cleanBuildPipelineVersion = getVersion(false).replace(/[^a-zA-Z0-9]/g, "_");
-function getCacheDirectory(root) {
-    const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
-    if (!fs.existsSync(tmpDir)) {
-        fs.mkdirSync(tmpDir, { recursive: true });
-    }
-    return tmpDir;
-}
-function hashObject(obj, depth = 0) {
-    if (obj instanceof Accessor) {
-        const buffer = obj.getArray();
-        return hashBuffer(buffer.buffer);
-    }
-    else if (obj == null) {
-        return 0;
-    }
-    if (typeof obj !== 'object') {
-        return hashString(String(obj));
-    }
-    try {
-        const str = JSON.stringify(obj);
-        return hashString(str);
-    }
-    catch (err) {
-        console.warn("WARN: [Cache] error during hashing of object:\n" + err);
-        if (debugLogs)
-            debugger;
-        // Fallback to a simple hash of the keys and values
-        // TODO: maybe we want to exclude this object now from the cache if it fails to serialize?
-        let fallbackHashj = 0;
-        if (typeof obj === 'object') {
-            if (obj === null)
-                return 0;
-            for (const key in obj) {
-                if (Object.prototype.hasOwnProperty.call(obj, key)) {
-                    const value = obj[key];
-                    fallbackHashj += hashString(key);
-                    if (typeof value === 'object') {
-                        if (depth > 10) {
-                            fallbackHashj += hashString("[object]");
-                        }
-                        else {
-                            fallbackHashj += hashObject(value, depth + 1);
-                        }
-                    }
-                    else {
-                        fallbackHashj += hashString(String(value));
-                    }
-                }
-            }
-        }
-        return fallbackHashj;
-    }
-}
-function hashBuffer(buffer, offset, length) {
-    if (hasher) {
-        const view = new Uint8Array(buffer, offset, length);
-        return hasher.update(view).digest();
-    }
-    let hash = 0;
-    const view = new Uint8Array(buffer, offset, length);
-    for (let i = 0; i < view.length; i++) {
-        hash = ((hash << 5) - hash) + view[i];
-        hash |= 0;
-    }
-    return hash;
-}
-function hashString(str) {
-    let hash = 0;
-    for (let i = 0; i < str.length; i++) {
-        hash = ((hash << 5) - hash) + str.charCodeAt(i);
-        hash |= 0;
-    }
-    return hash;
-}
+import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
+import tmp from 'tmp';
+import fs, { statSync } from 'fs';
+import path from 'path';
+import { getVersion } from "../utils/version.js";
+// import * as CHECKDISCSPACE from 'check-disk-space'
+import xxhash from "xxhash-wasm";
+let hasher = null;
+xxhash().then(module => {
+    hasher = module.create32();
+});
+/**
+ * Get the available space on the cache directory in bytes.
+ */
+// export async function getCacheAvailableSpaceOnDisc(): Promise<number> {
+//     try {
+//         const dir = getCacheDirectory();
+//         const res = await CHECKDISCSPACE.default(dir).catch(() => { return { free: 0 } });
+//         return res.free;
+//     }
+//     catch (e) {
+//         console.error(`Error checking disk space: ${e}`);
+//     }
+//     return 0;
+// }
+const buildPipelineVersion = getVersion(false);
+const debugLogs = process.argv.includes("--debug");
+/**
+ * Clear the cache.
+ * This will remove all files in the cache directory.
+ */
+export function clearCache(logger) {
+    const dir = getCacheDirectory(true);
+    fs.rmSync(dir, { recursive: true });
+    if (!logger)
+        console.log(`Cleared cache at ${dir}`);
+    else
+        logger.info(`Cleared cache at ${dir}`);
+}
+export const cacheSizeLimit = 1024 * 5;
+/**
+ * Ensure that the cache size does not exceed the given maximum size in MB.
+ * @param maxSizeMB The maximum size in MB.
+ */
+export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
+    const dir = getCacheDirectory(true);
+    console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}`);
+    let totalSize = 0;
+    const files = new Array();
+    function recurseDirectories(dir) {
+        fs.readdirSync(dir).forEach(file => {
+            const path = `${dir}/${file}`;
+            const stats = statSync(path);
+            if (stats.isDirectory()) {
+                recurseDirectories(path);
+            }
+            else {
+                totalSize += stats.size;
+                files.push({ path, size: stats.size });
+            }
+        });
+    }
+    recurseDirectories(dir);
+    const maxSizeInBytes = maxSizeMB * 1024 * 1024;
+    console.log(`Current cache size is ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
+    if (totalSize < maxSizeInBytes)
+        return;
+    files.sort((a, b) => {
+        const mtimeA = fs.statSync(a.path).mtimeMs;
+        const mtimeB = fs.statSync(b.path).mtimeMs;
+        return mtimeA - mtimeB;
+    });
+    const targetSize = maxSizeInBytes * 0.9;
+    // remove oldest files first
+    let filesRemoved = 0;
+    while (totalSize > targetSize) {
+        const file = files.shift();
+        if (!file)
+            break;
+        if (fs.statSync(file.path).isDirectory()) {
+            continue;
+        }
+        filesRemoved++;
+        fs.rmSync(file.path);
+        totalSize -= file.size;
+    }
+    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB (${filesRemoved} files removed)`);
+}
+/**
+ * Add data to the cache.
+ * @param key The key to store the data under.
+ * @param bytes The data to store as a Uint8Array.
+ */
+export function addToCache(key, bytes) {
+    const dir = getCacheDirectory();
+    const path = `${dir}/${key}.bin`;
+    if (debugLogs)
+        console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at \"${path}\"`);
+    if (bytes instanceof Buffer)
+        bytes = new Uint8Array(bytes);
+    fs.writeFileSync(path, bytes);
+}
+/**
+ * Try to get the data from the cache.
+ * @param key The key to look up.
+ * @returns The data as a Uint8Array or null if not found.
+ * @throws If the cache file is corrupted.
+ */
+export function tryGetFromCache(key) {
+    const dir = getCacheDirectory();
+    const path = `${dir}/${key}.bin`;
+    if (fs.existsSync(path)) {
+        const bytes = fs.readFileSync(path);
+        if (bytes instanceof Buffer)
+            return new Uint8Array(bytes);
+        return bytes;
+    }
+    return null;
+}
+/** Add meta data to the cache.
+ * @param key The key to store the meta data for.
+ * @param meta The meta data object to store. It will be serialized to JSON and stored in a file with the key as the name with the extension ".meta".
+ */
+export function addMetaToCache(key, meta) {
+    const dir = getCacheDirectory();
+    const metaPath = `${dir}/${key}.meta`;
+    meta['needle-build-pipeline:version'] = buildPipelineVersion;
+    fs.writeFileSync(metaPath, JSON.stringify(meta));
+}
+/** Try to get the meta data from the cache.
+ * @param key The key to look up.
+ * @param ignoreVersion If true, the version check is skipped. If false, the version must match the current build pipeline version.
+ * @returns The meta data object or null if not found.
+ */
+export function tryGetMetaFromCache(key, ignoreVersion = false) {
+    try {
+        const dir = getCacheDirectory();
+        const path = `${dir}/${key}.meta`;
+        if (fs.existsSync(path)) {
+            const bytes = fs.readFileSync(path);
+            if (bytes?.length > 0) {
+                const text = bytes.toString('utf-8');
+                if (text) {
+                    const obj = JSON.parse(text);
+                    if (ignoreVersion)
+                        return obj;
+                    if (obj['needle-build-pipeline:version'] === buildPipelineVersion)
+                        return obj;
+                }
+            }
+        }
+        return null;
+    }
+    catch (e) {
+        console.error(`Error reading cache meta: ${e.message}`);
+        return null;
+    }
+}
+/**
+ * Get the cache key for a property.
+ * @param property The property to get the key for.
+ * @returns The key as a string.
+ */
+export function getKey(property) {
+    let prefix = "prop";
+    if (property instanceof Texture) {
+        prefix = "image";
+    }
+    let hash = getHash(property);
+    const key = `${prefix}-${hash}`;
+    return key;
+}
+export function getKeyWithHash(property) {
+    let prefix = "prop";
+    if (property instanceof Texture) {
+        prefix = "image";
+    }
+    let hash = getHash(property);
+    const key = `${prefix}-${hash}`;
+    return { key, hash };
+}
+/**
+ * Get the hash for a property or an object.
+ * @param property The property to get the hash for.
+ * @param level The current level of recursion.
+ * @returns The hash as a number.
+ */
+export function getHash(property, level = 0) {
+    let hashStr = "";
+    let hash = 0;
+    if (property instanceof ExtensibleProperty) {
+        const extensions = property.listExtensions();
+        const extras = property.getExtras();
+        // const slots = listTextureSlots(doc, texture);
+        // const channels = getTextureChannelMask(doc, texture);
+        hash += hashObject({
+            extensions,
+            extras,
+        });
+        if (property instanceof Texture) {
+            const image = property.getImage();
+            const bytes = image?.buffer;
+            if (bytes)
+                hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
+        }
+        else if (property instanceof Mesh) {
+            const weights = property.getWeights();
+            hash += hashObject(weights);
+            const primitives = property.listPrimitives();
+            for (const prim of primitives) {
+                hash += hashString(getHash(prim, level + 1));
+            }
+        }
+        else if (property instanceof Primitive) {
+            const indices = property.getIndices();
+            const attributes = property.listAttributes();
+            const targets = property.listTargets();
+            // TODO: hash buffer view if possible
+            const semantics = property.listSemantics();
+            const semanticsNames = semantics.map(s => {
+                return {
+                    name: s,
+                };
+            });
+            hash += hashObject({
+                indices,
+                attributes,
+                targets,
+                semantics: semanticsNames
+            });
+            if (indices) {
+                const indexBuffer = indices.getArray();
+                if (indexBuffer) {
+                    hash += hashBuffer(indexBuffer.buffer, indexBuffer.byteOffset, indexBuffer.byteLength);
+                }
+            }
+            // We can not pass the whole buffer into hashObject since very large meshes will cause errors when trying to stringify
+            // See https://linear.app/needle/issue/NE-6414
+            for (const sem of semantics) {
+                const arr = property.getAttribute(sem)?.getArray();
+                if (arr) {
+                    hash += hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength);
+                }
+            }
+        }
+    }
+    else {
+        hash += hashObject(property);
+    }
+    return hashStr + hash;
+}
+const cleanBuildPipelineVersion = getVersion(false).replace(/[^a-zA-Z0-9]/g, "_");
+function getCacheDirectory(root) {
+    const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
+    if (!fs.existsSync(tmpDir)) {
+        fs.mkdirSync(tmpDir, { recursive: true });
+    }
+    return tmpDir;
+}
+function hashObject(obj, depth = 0) {
+    if (obj instanceof Accessor) {
+        const buffer = obj.getArray();
+        return hashBuffer(buffer.buffer);
+    }
+    else if (obj == null) {
+        return 0;
+    }
+    if (typeof obj !== 'object') {
+        return hashString(String(obj));
+    }
+    try {
+        const str = JSON.stringify(obj);
+        return hashString(str);
+    }
+    catch (err) {
+        console.warn("WARN: [Cache] error during hashing of object:\n" + err);
+        if (debugLogs)
+            debugger;
+        // Fallback to a simple hash of the keys and values
+        // TODO: maybe we want to exclude this object now from the cache if it fails to serialize?
+        let fallbackHashj = 0;
+        if (typeof obj === 'object') {
+            if (obj === null)
+                return 0;
+            for (const key in obj) {
+                if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                    const value = obj[key];
+                    fallbackHashj += hashString(key);
+                    if (typeof value === 'object') {
+                        if (depth > 10) {
+                            fallbackHashj += hashString("[object]");
+                        }
+                        else {
+                            fallbackHashj += hashObject(value, depth + 1);
+                        }
+                    }
+                    else {
+                        fallbackHashj += hashString(String(value));
+                    }
+                }
+            }
+        }
+        return fallbackHashj;
+    }
+}
+function hashBuffer(buffer, offset, length) {
+    if (hasher) {
+        const view = new Uint8Array(buffer, offset, length);
+        return hasher.update(view).digest();
+    }
+    let hash = 0;
+    const view = new Uint8Array(buffer, offset, length);
+    for (let i = 0; i < view.length; i++) {
+        hash = ((hash << 5) - hash) + view[i];
+        hash |= 0;
+    }
+    return hash;
+}
+function hashString(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = ((hash << 5) - hash) + str.charCodeAt(i);
+        hash |= 0;
+    }
+    return hash;
+}

package/dist/cache/index.d.ts

@@ -1 +1 @@
-export * from "./cache.js";
+export * from "./cache.js";

package/dist/cache/index.js

@@ -1 +1 @@
-export * from "./cache.js";
+export * from "./cache.js";

package/dist/cli/index.d.ts

@@ -1,2 +1,2 @@
-#! /usr/bin/env node
-export {};
+#! /usr/bin/env node
+export {};